Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio sd.cpp: a lost Vulkan device is not a strike; Arc / ROCm keep Vulkan; a UHD iGPU beside a card no longer vetoes Vulkan (#14610)

Daniel Hanf4a21e236fa4
diff --git a/studio/backend/core/inference/diffusion_engine_router.py b/studio/backend/core/inference/diffusion_engine_router.py
index 53092e865a7..f33a6b383ff 100644
--- a/studio/backend/core/inference/diffusion_engine_router.py
+++ b/studio/backend/core/inference/diffusion_engine_router.py
@@ -240,6 +240,31 @@ def _accessible_render_vendors(vendors) -> set:
return found
+def _reachable_intel_arc_render_node() -> bool:
+ """True when this process can open the render node of an Intel Arc itself (PCI class per node), not
+ only some Intel node: beside a UHD iGPU, a container may mount the iGPU's node alone."""
+ import glob
+
+ from utils.hardware.hardware import _intel_pci_device_is_xpu_class
+
+ for node in glob.glob("/dev/dri/renderD*"):
+ if not os.access(node, os.R_OK | os.W_OK):
+ continue
+ device = f"/sys/class/drm/{os.path.basename(node)}/device"
+ try:
+ with open(f"{device}/vendor", encoding = "utf-8") as f:
+ if _RENDER_NODE_VENDORS.get(f.read().strip().lower()) != "intel":
+ continue
+ except OSError:
+ continue
+ try:
+ if _intel_pci_device_is_xpu_class(device) is True:
+ return True
+ except Exception: # noqa: BLE001 -- unknown class: not proof the Arc is reachable
+ continue
+ return False
+
+
def _vulkan_gpu_for_cpu_torch() -> bool:
"""True when torch is CPU-only but the host has a GPU the Vulkan sd.cpp build can render on:
an AMD card (Vega, RDNA1 and Polaris get no ROCm wheel) or an Intel Arc. Never beside NVIDIA,
@@ -251,7 +276,7 @@ def _vulkan_gpu_for_cpu_torch() -> bool:
# ggml reads this one itself; anything but a list of ordinals selects nothing or throws. Its
# ordinals index the raw Vulkan list, so it never vouches for which card a mixed host gets.
vk_mask = os.environ.get("GGML_VK_VISIBLE_DEVICES")
- if vk_mask is not None and not re.fullmatch(r"\s*\d+\s*(,\s*\d+\s*)*", vk_mask):
+ if vk_mask is not None and not re.fullmatch(r"\s*\d+(?:[\s,]+\d+)*[\s,]*", vk_mask):
return False
inventory = _physical_inventory()
if inventory.get("unknown") or "nvidia" in (inventory.get("unanswered") or ()):
@@ -278,12 +303,23 @@ def _vulkan_gpu_for_cpu_torch() -> bool:
reachable = _accessible_render_vendors({d.get("vendor") for d in gpus})
gpus = [d for d in gpus if d.get("vendor") in reachable]
masked = _masked_vendors()
- # Unpinned sd.cpp takes the first Vulkan device, which may be a UHD iGPU beside the card.
if not gpus or any(d.get("vendor") in masked for d in gpus):
return False
- if not all(_renders(d) for d in gpus):
- return False
- return _hardware_vulkan_driver({d.get("vendor") for d in gpus}) and _vulkan_loader_loads()
+ rendering = [d for d in gpus if _renders(d)]
+ if len(rendering) < len(gpus):
+ # Unpinned sd.cpp takes a discrete GPU before an iGPU (get_default_backend_name); a mask's ordinals may name the iGPU.
+ if vk_mask is not None or not rendering:
+ return False
+ if any(d.get("vendor") != "intel" for d in gpus if not _renders(d)):
+ return False
+ # The vendor-level node check cannot tell an Arc's node from the iGPU's: an unopenable Arc vouches for nothing.
+ if sys.platform != "win32" and not _reachable_intel_arc_render_node():
+ rendering = [d for d in rendering if d.get("vendor") != "intel"]
+ if not rendering:
+ return False
+ # The driver must belong to a card that renders: beside an AMD card, an Intel-only ICD drives just the iGPU.
+ rendering_vendors = {d.get("vendor") for d in rendering}
+ return _hardware_vulkan_driver(rendering_vendors) and _vulkan_loader_loads()
# The ICD manifests that drive each vendor's cards (RADV as radeon_icd or radv_icd, AMDVLK /
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index bb9e4e661f2..ddc661581af 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -644,7 +644,7 @@ def sd_cpp_device_name_for_ordinal(binary: Optional[str], ordinal: Optional[int]
choice, i.e. today's behaviour."""
if not binary or ordinal is None:
return None
- text = _sd_cpp_probe_output(binary, "--list-devices")
+ text = sd_cpp_device_listing(binary)
if text is not None:
for line in text.splitlines():
name = line.split("\t", 1)[0].strip()
@@ -673,8 +673,21 @@ _CUDA_DEVICE_CC_RE = re.compile(
)
-# Last --list-devices answer per binary + (size, mtime_ns); only the capability read reuses it, never the verdict.
+# Last --list-devices answer per binary. The accelerator verdict always re-probes (a load re-vets the listing on purpose).
_LAST_DEVICE_LISTING: dict = {}
+# What ggml reads to pick or hide devices.
+_DEVICE_LISTING_ENV_VARS: tuple[str, ...] = (
+ "GGML_VK_VISIBLE_DEVICES",
+ "VK_ICD_FILENAMES",
+ "VK_DRIVER_FILES",
+ "VK_ADD_DRIVER_FILES",
+ "VK_LOADER_DRIVERS_SELECT",
+ "VK_LOADER_DRIVERS_DISABLE",
+ "CUDA_VISIBLE_DEVICES",
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "GPU_DEVICE_ORDINAL",
+)
def _binary_stat_identity(binary: str) -> Optional[tuple[int, int]]:
@@ -685,13 +698,49 @@ def _binary_stat_identity(binary: str) -> Optional[tuple[int, int]]:
return (st.st_size, st.st_mtime_ns)
+def _hardware_inventory_generation() -> Optional[int]:
+ try:
+ from utils.hardware.hardware import physical_gpu_inventory_generation
+ return physical_gpu_inventory_generation()
+ except Exception: # noqa: BLE001 -- an identity read must never fail the probe
+ return None
+
+
+def _device_listing_identity(binary: str) -> tuple:
+ return (
+ _binary_stat_identity(binary),
+ tuple(os.environ.get(name) for name in _DEVICE_LISTING_ENV_VARS),
+ _hardware_inventory_generation(),
+ )
+
+
def _remember_device_listing(binary: Optional[str], text: Optional[str]) -> None:
if not binary:
return
if text is None:
- # Keep the last good listing; readers check (size, mtime_ns), so a changed binary's listing is never served.
+ # An unreadable probe or a timeout is never cached.
return
- _LAST_DEVICE_LISTING[binary] = (_binary_stat_identity(binary), text)
+ _LAST_DEVICE_LISTING[binary] = (_device_listing_identity(binary), text)
+
+
+def _cached_device_listing(binary: str, *, require_file: bool = False) -> Optional[str]:
+ seen = _LAST_DEVICE_LISTING.get(binary)
+ if seen is None:
+ return None
+ identity = _device_listing_identity(binary)
+ if require_file and identity[0] is None:
+ return None
+ return seen[1] if seen[0] == identity else None
+
+
+def sd_cpp_device_listing(binary: str) -> Optional[str]:
+ """``binary --list-devices``, reusing the listing already taken of this file under this environment."""
+ text = _cached_device_listing(binary, require_file = True)
+ if text is None:
+ text = _sd_cpp_probe_output(binary, "--list-devices")
+ if _binary_stat_identity(binary) is not None:
+ _remember_device_listing(binary, text)
+ return text
def sd_cpp_cuda_compute_capability(
@@ -705,12 +754,9 @@ def sd_cpp_cuda_compute_capability(
only reads the listing the last accelerator verdict took of this file."""
if not binary:
return None
- text = None
- seen = _LAST_DEVICE_LISTING.get(binary)
- if seen is not None and seen[0] == _binary_stat_identity(binary):
- text = seen[1]
- elif probe:
- text = _sd_cpp_probe_output(binary, "--list-devices")
+ text = _cached_device_listing(binary)
+ if text is None and probe:
+ text = sd_cpp_device_listing(binary)
if text is None or not _CUDA_INIT_RE.search(text):
return None
caps = {
@@ -730,8 +776,7 @@ def sd_cpp_cuda_compute_capability(
def sd_cpp_device_listing_known(binary: Optional[str]) -> bool:
if not binary:
return False
- seen = _LAST_DEVICE_LISTING.get(binary)
- return seen is not None and seen[0] == _binary_stat_identity(binary)
+ return _cached_device_listing(binary) is not None
# Runs when --list-devices timed out (possibly a stalled driver); CUDA calls have no deadline, so bound the wait.
@@ -847,12 +892,9 @@ def sd_cpp_vulkan_device_is_uma(
memory for such a device sums every heap, the GTT the pinned weights come from included. None when unsure."""
if not binary:
return None
- text = None
- seen = _LAST_DEVICE_LISTING.get(binary)
- if seen is not None and seen[0] == _binary_stat_identity(binary):
- text = seen[1]
- elif probe:
- text = _sd_cpp_probe_output(binary, "--list-devices")
+ text = _cached_device_listing(binary)
+ if text is None and probe:
+ text = sd_cpp_device_listing(binary)
if text is None:
return None
uma = {int(m.group(1)): m.group(2) == "1" for m in _VULKAN_DEVICE_UMA_RE.finditer(text)}
@@ -1072,7 +1114,7 @@ def sd_cpp_device_named(
wanted = _normalized_card_name(card_name)
if not wanted:
return None
- text = listing if listing is not None else _sd_cpp_probe_output(binary, "--list-devices")
+ text = listing if listing is not None else sd_cpp_device_listing(binary)
if text is None:
return None
# Both readings: containment cannot tell `RX 7600` from `RX 7600 XT`, equality cannot see through a driver tag.
@@ -1326,6 +1368,16 @@ def _note_failed_upgrade(accelerator: str) -> None:
_ACCELERATOR_FALLBACK: dict[str, str] = {"rocm": "vulkan", "vulkan": "cpu"}
+def _vulkan_cpu_rung_applies() -> bool:
+ """Whether ambiguous failures can move the Vulkan build to CPU: only on the CPU-torch host routed to Vulkan. An XPU
+ host (always Vulkan) and a ROCm host on its Vulkan rung leave the GPU only on decisive evidence. Unknown: as before."""
+ try:
+ from utils.hardware.hardware import DeviceType, get_device
+ return get_device() == DeviceType.CPU
+ except Exception: # noqa: BLE001
+ return True
+
+
# The sonames the upstream ROCm archive imports and does not ship.
_ROCM_RUNTIME_SONAMES: tuple[str, ...] = (
"libamdhip64.so.7",
@@ -1436,12 +1488,9 @@ def sd_cpp_listed_device(
file (no subprocess)."""
if not binary:
return None, None
- text = None
- seen = _LAST_DEVICE_LISTING.get(binary)
- if seen is not None and seen[0] == _binary_stat_identity(binary):
- text = seen[1]
- elif probe:
- text = _sd_cpp_probe_output(binary, "--list-devices")
+ text = _cached_device_listing(binary)
+ if text is None and probe:
+ text = sd_cpp_device_listing(binary)
if text is None:
return None, None
for line in text.splitlines():
@@ -1492,6 +1541,47 @@ _accelerator_runtime_failures: dict[str, dict] = {}
# AMBIGUOUS failures needed to divert a host (a decisive one is enough alone).
_AMBIGUOUS_FAILURE_STRIKES = 2
+# Rule 2 stopped counting Vulkan device-lost errors: an older Vulkan record keeps only its decisive evidence.
+_FAILURE_RULE = 2
+
+
+def _record_under_current_rule(klass: str, record: Optional[dict]) -> Optional[dict]:
+ if not isinstance(record, dict) or klass != "vulkan":
+ return record
+ if int(record.get("rule", 1) or 1) >= _FAILURE_RULE:
+ return record
+ return _decisive_evidence_only(record)
+
+
+def _decisive_evidence_only(record: Optional[dict]) -> Optional[dict]:
+ """*record* with its ambiguous strikes dropped, or None when nothing decisive is left."""
+ if not isinstance(record, dict):
+ return record
+ proven_cards = {
+ name: {"strikes": 0, "proven": True}
+ for name, entry in (record.get("per_card") or {}).items()
+ if isinstance(entry, dict) and entry.get("proven")
+ }
+ unscoped = _unscoped_evidence(record)
+ # Proof without per-card tallies stays as written: record-wide, and scoped to its ``cards`` when it names any.
+ whole = bool(record.get("proven")) and not isinstance(record.get("per_card"), dict)
+ if not (proven_cards or unscoped["proven"] or whole):
+ return None
+ kept = {
+ "strikes": 0,
+ "proven": True,
+ "fingerprint": record.get("fingerprint") or {},
+ "rule": _FAILURE_RULE,
+ }
+ if record.get("cards"):
+ # A card left in ``cards`` without its own entry would divert on another card's record-wide proof.
+ kept["cards"] = [
+ c for c in record["cards"] if whole or unscoped["proven"] or c in proven_cards
+ ]
+ if proven_cards:
+ kept["per_card"] = proven_cards
+ kept["unscoped"] = {"strikes": 0, "proven": unscoped["proven"]}
+ return kept
def _accelerator_class_of(accelerator: Optional[str]) -> str:
@@ -1662,10 +1752,15 @@ def _normalise_failure_record(key: str, value: object) -> Optional[dict]:
except (TypeError, ValueError):
strikes = 0
fingerprint = value.get("fingerprint")
+ try:
+ rule = int(value.get("rule", 1) or 1)
+ except (TypeError, ValueError):
+ rule = 1
record = {
"strikes": max(strikes, 0),
"proven": bool(value.get("proven", False)),
"fingerprint": fingerprint if isinstance(fingerprint, dict) else {},
+ "rule": rule,
}
cards = [str(card).strip() for card in (value.get("cards") or []) if str(card).strip()]
if cards:
@@ -1778,7 +1873,7 @@ def note_accelerator_runtime_failure(
fingerprint = dict(fingerprint) if isinstance(fingerprint, dict) else _accelerator_fingerprint()
records = _stored_accelerator_runtime_failures()
records.update({k: v for k, v in _accelerator_runtime_failures.items() if k not in records})
- previous = records.get(klass)
+ previous = _record_under_current_rule(klass, records.get(klass))
if previous is not None and not _fingerprint_still_applies(
previous.get("fingerprint"), fingerprint
):
@@ -1816,6 +1911,7 @@ def note_accelerator_runtime_failure(
# The union, for a caller that names no card and for the settings report.
"proven": bool(proven) or bool((previous or {}).get("proven", False)),
"fingerprint": fingerprint,
+ "rule": _FAILURE_RULE,
}
if cards:
record["cards"] = cards
@@ -1877,9 +1973,24 @@ def accelerator_runtime_failed(accelerator: Optional[str], card: Optional[str] =
if not klass:
return False
fingerprint = _accelerator_fingerprint()
- if _record_diverts(_accelerator_runtime_failures.get(klass), fingerprint, card):
+ if _record_diverts(
+ _applicable_record(klass, _accelerator_runtime_failures.get(klass)), fingerprint, card
+ ):
return True
- return _record_diverts(_stored_accelerator_runtime_failures().get(klass), fingerprint, card)
+ return _record_diverts(
+ _applicable_record(klass, _stored_accelerator_runtime_failures().get(klass)),
+ fingerprint,
+ card,
+ )
+
+
+def _applicable_record(klass: str, record: Optional[dict]) -> Optional[dict]:
+ """The evidence in *record* that still counts on this host: old-rule ambiguous Vulkan strikes are
+ dropped, and outside the CPU-torch route only decisive Vulkan evidence diverts."""
+ record = _record_under_current_rule(klass, record)
+ if klass == "vulkan" and not _vulkan_cpu_rung_applies():
+ return _decisive_evidence_only(record)
+ return record
def off_torch_build_mismatch(off_torch: Any, binary: Optional[str]) -> Optional[str]:
@@ -1952,7 +2063,8 @@ def accelerator_runtime_failure_state() -> dict:
"fallback": _ACCELERATOR_FALLBACK.get(klass),
"strikes": int(record.get("strikes", 0) or 0),
"proven": bool(record.get("proven", False)),
- "diverting": enabled and _record_diverts(record, fingerprint),
+ "diverting": enabled
+ and _record_diverts(_applicable_record(klass, record), fingerprint),
"stale": not _fingerprint_still_applies(record.get("fingerprint"), fingerprint),
}
)
@@ -1991,8 +2103,7 @@ _ACCELERATOR_AMBIGUOUS_FAILURE_MARKERS: tuple[str, ...] = (
"rocm error",
"rocblas error",
"memory access fault by gpu node",
- "errordevicelost",
- "vk_error_device_lost",
+ # Not VK_ERROR_DEVICE_LOST: the Vulkan spec makes it recoverable by recreating the device (TDR, scheduler reset).
"errorinitializationfailed",
"vk_error_initialization_failed",
)
@@ -2060,6 +2171,34 @@ def output_shows_capacity_failure(text: Optional[str]) -> bool:
return any(marker in lowered for marker in _ACCELERATOR_CAPACITY_FAILURE_MARKERS)
+# A lost Vulkan device: recoverable by creating a new device, so neither a build strike nor a fatal error.
+_VULKAN_DEVICE_LOST_MARKERS: tuple[str, ...] = ("errordevicelost", "vk_error_device_lost")
+
+
+def output_shows_vulkan_device_lost(text: Optional[str]) -> bool:
+ if not text:
+ return False
+ lowered = str(text).lower()
+ return any(marker in lowered for marker in _VULKAN_DEVICE_LOST_MARKERS)
+
+
+# Appended when a lost Vulkan device repeats and no CPU fallback applies. Must match no failure marker.
+_REPEATED_DEVICE_LOSS_HINT = (
+ "The GPU lost its Vulkan device twice in a row (usually a driver reset during the render). Try Low VRAM "
+ "memory mode, a smaller resolution or a smaller quant, or load the model again to retry on the GPU."
+)
+
+
+def _device_loss_cpu_fallback_applies(state: Any) -> bool:
+ """Whether a repeated Vulkan device loss may move this load to the CPU backend: only the CPU-torch Vulkan
+ route (the one with a CPU rung, as on main), the Vulkan-to-CPU fallback enabled, and no user-picked GPU."""
+ if getattr(state, "device", None) != "vulkan" or getattr(state, "off_torch_device", None):
+ return False
+ if getattr(state, "gpu_explicit", False):
+ return False
+ return fallback_accelerator_for("vulkan") == "cpu" and _vulkan_cpu_rung_applies()
+
+
def output_shows_accelerator_failure(text: Optional[str]) -> bool:
"""True when sd-cli output names a failure of the GPU BUILD rather than of the request; an unrecognised failure persists nothing."""
if not text:
@@ -2734,6 +2873,8 @@ class _SdState:
# UNSLOTH_DIFFUSION_SD_CPP_DEVICE placement: its label, and the env every spawn gets to open only that card.
off_torch_device: Optional[str] = None
child_env: tuple[tuple[str, str], ...] = ()
+ # The user picked the GPU (a gpu_ids pool or UNSLOTH_DIFFUSION_SD_CPP_DEVICE): never moved to CPU behind their back.
+ gpu_explicit: bool = False
def spawn_env(self) -> Optional[dict[str, str]]:
return dict(self.child_env) or None
@@ -3071,6 +3212,9 @@ class SdCppDiffusionBackend:
# sd-server started for an in-flight load, before it commits to _state; tracked so an unload can stop it
# mid-startup.
self._pending_server: Optional[SdCppServer] = None
+ # Server a generation is stopping for a replacement: status() must not reap it as an idle crash, or the
+ # handover reads an unload and kills the healthy replacement.
+ self._replacing_server: Optional[SdCppServer] = None
self._gen: Optional[_SdGen] = None
# Set by _resolve_backend when it had to skip an accelerator install because the managed tree was still in
# use; the load retries it once the tree is free.
@@ -3361,6 +3505,7 @@ class SdCppDiffusionBackend:
from core.inference.diffusion_engine_router import off_torch_sd_cpp_device
off_torch = off_torch_sd_cpp_device()
+ gpu_explicit = bool(gpu_ids) or off_torch is not None
if off_torch is not None:
gpu_ids, gpu_ordinal = None, None
# Same fallback the diffusers and video backends take: the route ranks the selection and passes the winner,
@@ -3457,6 +3602,7 @@ class SdCppDiffusionBackend:
memory_mode = memory_mode,
speed_mode = speed_mode,
gpu_ordinal = gpu_ordinal,
+ gpu_explicit = gpu_explicit,
off_torch = off_torch,
_load_token = token,
_cancel_event = cancel_event,
@@ -3484,6 +3630,7 @@ class SdCppDiffusionBackend:
memory_mode: Optional[str] = None,
speed_mode: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
+ gpu_explicit: bool = False,
off_torch: Any = None,
_load_token: int,
_cancel_event: Optional[threading.Event] = None,
@@ -3950,6 +4097,7 @@ class SdCppDiffusionBackend:
selected_card = self._loading_card,
off_torch_device = off_torch.label if off_torch is not None else None,
child_env = tuple(sorted((spawn_env or {}).items())),
+ gpu_explicit = gpu_explicit,
)
superseded = False
orphan: Optional[SdCppServer] = None
@@ -4890,6 +5038,7 @@ class SdCppDiffusionBackend:
base_seed = int(seed) & ((1 << 63) - 1)
images: list = []
seeds: list[int] = []
+
# Stage LoRAs into a per-request subdir of the server lora-model-dir (so a prior request's adapters cannot
# leak in); removed after.
lora_payload: Optional[list[dict]] = None
@@ -4913,6 +5062,9 @@ class SdCppDiffusionBackend:
# images per job, so each chunk gets whatever is left rather than its own full budget and a long batch still
# ends on time.
deadline = time.monotonic() + NATIVE_GENERATION_TIMEOUT_S
+ # One same-build respawn per request; a second loss moves this load to CPU where the rung applies. Persists nothing.
+ device_respawned = False
+ device_losses = 0
hires_payload: Optional[dict] = None
hires_staged: list[Path] = []
@@ -4981,34 +5133,68 @@ class SdCppDiffusionBackend:
cancel_event = cancel,
total_timeout = max(deadline - time.monotonic(), 1.0),
)
- except RuntimeError as exc:
+ except RuntimeError as first_exc:
# A ggml unsupported-op abort killed the server: this graph cannot run on the GPU backend at all,
- # so restart the model on the CPU backend once and retry this chunk. Any other death propagates.
- server = self._restart_server_on_cpu_backend(state, str(exc), cancel)
- if server is None:
- raise
- state = replace(state, server = server)
- with self._lock:
- if self._state is not None and self._state.server is not None:
- self._state = state
- if lora_payload is not None:
- # The replacement server has its own scratch dir: stage the LoRAs there again.
- lora_payload = _stage_loras(server)
- if lora_payload is None:
- raise RuntimeError(
- "The CPU-backend native server has no LoRA directory for the selected LoRAs."
+ # so restart the model on the CPU backend once and retry this chunk. A lost Vulkan device needs a
+ # fresh device: respawn the same build once. Any other death propagates.
+ exc = first_exc
+ while True:
+ moved_to_cpu = False
+ server = self._restart_server_on_cpu_backend(state, str(exc), cancel)
+ if server is None and output_shows_vulkan_device_lost(str(exc)):
+ device_losses += 1
+ if device_losses >= 2:
+ server = self._restart_server_on_cpu_backend(
+ state, str(exc), cancel, after_device_loss = True
+ )
+ if server is None:
+ raise RuntimeError(
+ f"{exc} {_REPEATED_DEVICE_LOSS_HINT}"
+ ) from exc
+ moved_to_cpu = True
+ elif not device_respawned:
+ server = self._respawn_server_after_device_loss(
+ state, str(exc), cancel
+ )
+ device_respawned = server is not None
+ if server is None:
+ raise exc
+ old_server = state.server
+ state = replace(state, server = server)
+ if moved_to_cpu:
+ # Placement follows the server: the flags the CPU server got, no Vulkan pin.
+ state = replace(
+ state,
+ device = "cpu",
+ offload_flags = tuple(
+ without_device_backend_flags(state.offload_flags)
+ ),
+ physical_gpu_id = None,
)
- payload["lora"] = list(lora_payload)
- if hires_plan is not None:
- # The replacement server has its own scratch dir: stage the upscaler there again.
- hires_payload = _stage_hires(server)
- payload["hires"] = dict(hires_payload)
- blobs = server.img_gen(
- payload,
- on_step = self._on_log,
- cancel_event = cancel,
- total_timeout = max(deadline - time.monotonic(), 1.0),
- )
+ self._hand_over_replacement(state, server, old_server)
+ # The old server's stop removed its scratch dir, and the staged adapters with it.
+ if lora_payload is not None:
+ lora_payload = _stage_loras(server)
+ if lora_payload is None:
+ raise RuntimeError(
+ "The replacement native server has no LoRA directory for the selected LoRAs."
+ )
+ payload["lora"] = list(lora_payload)
+ if hires_plan is not None:
+ # The replacement has its own scratch dir: stage the upscaler there again.
+ hires_payload = _stage_hires(server)
+ payload["hires"] = dict(hires_payload)
+ try:
+ blobs = server.img_gen(
+ payload,
+ on_step = self._on_log,
+ cancel_event = cancel,
+ total_timeout = max(deadline - time.monotonic(), 1.0),
+ )
+ break
+ except RuntimeError as retry_exc:
+ exc = retry_exc
+ device_losses = 0
# All-or-nothing per chunk; a layered generation decodes layers + 1 images.
expected = count * ((layers + 1) if layers else 1)
if not cancel.is_set() and len(blobs) != expected:
@@ -5034,8 +5220,117 @@ class SdCppDiffusionBackend:
pass
return images, seeds
- def _restart_server_on_cpu_backend(
+ def _begin_replacement(self, state: _SdState) -> bool:
+ """Mark ``state.server`` as being replaced, under ``_lock``, before the caller stops it, so status() reports
+ the load as still loaded instead of reaping it. False when an unload already took the state: no restart."""
+ with self._lock:
+ if self._state is None or self._state.server is not state.server:
+ return False
+ self._replacing_server = state.server
+ return True
+
+ def _end_replacement(self, old_server: Any) -> None:
+ with self._lock:
+ if self._replacing_server is old_server:
+ self._replacing_server = None
+
+ def _hand_over_replacement(self, state: _SdState, server: SdCppServer, old_server: Any) -> None:
+ """Publish a restarted sd-server into ``_state`` in the same locked step that drops it from
+ ``_pending_server``, so an unload always sees it in one of the two. Only while ``_state`` still holds the
+ server being replaced: when an unload cleared it or a superseding load replaced it, stop the replacement
+ and cancel instead of generating on an untracked process or overwriting the newer load."""
+ with self._lock:
+ if self._pending_server is server:
+ self._pending_server = None
+ if self._replacing_server is old_server:
+ self._replacing_server = None
+ published = self._state is not None and self._state.server is old_server
+ if published:
+ self._state = state
+ else:
+ self._reserve_stop()
+ if not published:
+ self._stop_reserved(server)
+ raise SdCppCancelled("sd-server generation was cancelled.")
+
+ def _start_replacement(
+ self, state: _SdState, server: SdCppServer, cancel: threading.Event, **start_kwargs: Any
+ ) -> Optional[SdCppServer]:
+ """Start a published-as-pending replacement sd-server for this generation; None when it fails.
+
+ The start blocks for the model load (up to its startup timeout) while the caller holds
+ ``_generate_lock``. ``cancel_generate()`` and a superseding load only set ``cancel``, so a watcher stops
+ the pending server when that happens: the readiness wait bails at once and the generation raises
+ ``SdCppCancelled`` instead of loading a replacement nobody wants."""
+ done = threading.Event()
+
+ def _watch() -> None:
+ while not done.is_set():
+ if cancel.wait(0.25):
+ server.stop()
+ return
+
+ watcher = threading.Thread(target = _watch, name = "sd-server-replacement-cancel", daemon = True)
+ watcher.start()
+ failed = False
+ try:
+ server.start(state.files, **start_kwargs)
+ except Exception: # noqa: BLE001 -- the original server error is the more useful one
+ failed = True
+ finally:
+ done.set()
+ watcher.join()
+ if failed or cancel.is_set():
+ with self._lock:
+ if self._pending_server is server:
+ self._pending_server = None
+ server.stop()
+ self._end_replacement(state.server)
+ if cancel.is_set():
+ raise SdCppCancelled("sd-server generation was cancelled.")
+ return None
+ # Still pending on success: _hand_over_replacement publishes it, so an unload never misses it.
+ return server
+
+ def _respawn_server_after_device_loss(
self, state: _SdState, error_text: str, cancel: threading.Event
+ ) -> Optional[SdCppServer]:
+ """Restart this checkpoint's sd-server on the same build after a lost Vulkan device; None if that
+ does not apply. The Vulkan spec makes VK_ERROR_DEVICE_LOST recoverable only by creating a new device,
+ which the running server never does, so every later job on it would fail. No build strike: a lost
+ device (a TDR or a scheduler reset) says nothing about whether this build runs here."""
+ if not output_shows_vulkan_device_lost(error_text):
+ return None
+ if state.server is None or cancel.is_set():
+ return None
+ if not self._begin_replacement(state):
+ return None
+ logger.warning(
+ "sd-server lost its Vulkan device; restarting it on the same build. Details: %s",
+ error_text[:300],
+ )
+ state.server.stop()
+ server = SdCppServer(state.server.binary)
+ with self._lock:
+ self._pending_server = server
+ return self._start_replacement(
+ state,
+ server,
+ cancel,
+ vae_format = state.vae_format,
+ offload = state.offload_flags,
+ native_speed = state.native_speed,
+ threads = state.threads,
+ env = state.spawn_env(),
+ )
+
+ def _restart_server_on_cpu_backend(
+ self,
+ state: _SdState,
+ error_text: str,
+ cancel: threading.Event,
+ *,
+ after_device_loss: bool = False,
) -> Optional[SdCppServer]:
"""Relaunch this checkpoint's sd-server with ``--backend cpu``; None if that does not apply.
@@ -5051,8 +5346,15 @@ class SdCppDiffusionBackend:
(``--offload-to-cpu`` moves parameters, not compute), so the restart runs the same
checkpoint slower rather than not at all. Done once per load: the second abort, or any other
cause of death, is surfaced to the caller.
+
+ ``after_device_loss``: the respawned server lost its Vulkan device too (a host whose driver resets the
+ card on every render). Only on the CPU-torch Vulkan route, the one route with a CPU rung, and never for a
+ GPU the user picked. For this load only and nothing is persisted: the next load tries Vulkan again.
"""
- if not is_ggml_unsupported_op_abort(error_text):
+ if after_device_loss:
+ if not _device_loss_cpu_fallback_applies(state):
+ return None
+ elif not is_ggml_unsupported_op_abort(error_text):
return None
if self._cpu_backend_forced or state.device == "cpu":
return None # already on CPU: the abort is not a backend-placement problem
@@ -5061,38 +5363,40 @@ class SdCppDiffusionBackend:
server_binary = find_sd_server_binary()
if not server_binary:
return None
- logger.warning(
- "sd-server aborted on an op the '%s' backend cannot run; restarting on the CPU "
- "backend (slower, but it completes). Details: %s",
- state.device,
- error_text[:300],
- )
+ if not self._begin_replacement(state):
+ return None
+ if after_device_loss:
+ logger.warning(
+ "sd-server lost its Vulkan device again after a restart; this model now renders on the CPU "
+ "backend (slower) until it is loaded again, which tries Vulkan first. Details: %s",
+ error_text[:300],
+ )
+ else:
+ logger.warning(
+ "sd-server aborted on an op the '%s' backend cannot run; restarting on the CPU "
+ "backend (slower, but it completes). Details: %s",
+ state.device,
+ error_text[:300],
+ )
self._cpu_backend_forced = True
state.server.stop()
server = SdCppServer(server_binary)
with self._lock:
self._pending_server = server
- try:
- server.start(
- state.files,
- vae_format = state.vae_format,
- # WITHOUT the device pin. sd.cpp joins repeated --backend values into one spec instead of replacing,
- # and an explicit diffusion=CUDA0 outranks the bare `cpu` default, so leaving the pin on would restart
- # the server onto the very backend that just aborted.
- offload = without_device_backend_flags(state.offload_flags),
- native_speed = state.native_speed,
- threads = state.threads,
- env = state.spawn_env(),
- extra_args = list(CPU_BACKEND_FLAGS),
- )
- except Exception: # noqa: BLE001 -- the original abort is the more useful error
- server.stop()
- return None
- finally:
- with self._lock:
- if self._pending_server is server:
- self._pending_server = None
- return server
+ return self._start_replacement(
+ state,
+ server,
+ cancel,
+ vae_format = state.vae_format,
+ # WITHOUT the device pin. sd.cpp joins repeated --backend values into one spec instead of replacing,
+ # and an explicit diffusion=CUDA0 outranks the bare `cpu` default, so leaving the pin on would restart
+ # the server onto the very backend that just aborted.
+ offload = without_device_backend_flags(state.offload_flags),
+ native_speed = state.native_speed,
+ threads = state.threads,
+ env = state.spawn_env(),
+ extra_args = list(CPU_BACKEND_FLAGS),
+ )
def _generate_oneshot(
self,
@@ -5317,6 +5621,7 @@ class SdCppDiffusionBackend:
self._active_generate_cancel.set()
state = self._state
self._state = None
+ self._replacing_server = None
self._load_token += 1
self._loading = None
# Grab a mid-start() uncommitted server too so we can stop it (startup is abortable).
@@ -5350,13 +5655,17 @@ class SdCppDiffusionBackend:
state is not None
and state.mode == "server"
and state.server is not None
+ and state.server is not self._replacing_server
and not state.server.is_alive()
):
- logger.warning("sd-server exited after load; clearing loaded state")
with self._lock:
- if self._state is state:
+ # Re-checked under the lock: a generation stopping it for a replacement marks it first.
+ reap = self._state is state and state.server is not self._replacing_server
+ if reap:
+ logger.warning("sd-server exited after load; clearing loaded state")
self._state = None
- state = None
+ if reap:
+ state = None
if state is None:
return {
"loaded": False,
diff --git a/studio/backend/tests/test_diffusion_hires.py b/studio/backend/tests/test_diffusion_hires.py
index 56cc0dc9625..dafed138475 100644
--- a/studio/backend/tests/test_diffusion_hires.py
+++ b/studio/backend/tests/test_diffusion_hires.py
@@ -930,8 +930,11 @@ def test_native_cpu_restart_restages_the_model_upscaler(tmp_path, monkeypatch):
target = (96, 96),
refine_steps = 4,
)
+ state = _State(family = _fam("sdxl"), server = _Server(old_dir, fail = True))
+ # The replacement is published only while the backend still holds the server it replaces.
+ b._state = state
images, _ = b._generate_server(
- _State(family = _fam("sdxl"), server = _Server(old_dir, fail = True)),
+ state,
prompt = "p",
negative_prompt = None,
width = 64,
@@ -953,6 +956,7 @@ def test_native_cpu_restart_restages_the_model_upscaler(tmp_path, monkeypatch):
new_dir,
] # staged again in the new server's dir
assert not any(p.exists() for p in staged_paths) # both removed after
+ assert b._state.server is new_server
def test_diffusers_status_names_why_hires_is_hidden():
diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.py
index d938c4ce07b..9a5d697fdcd 100644
--- a/studio/backend/tests/test_sd_cpp_backend.py
+++ b/studio/backend/tests/test_sd_cpp_backend.py
@@ -5,9 +5,12 @@
from __future__ import annotations
+import os
+import shutil
import threading
-from pathlib import Path
+import time
import types
+from pathlib import Path
import pytest
from PIL import Image
@@ -2145,10 +2148,14 @@ def _server_lora_run(monkeypatch, tmp_path, *, fail_first):
new_dir.mkdir()
b = SdCppDiffusionBackend()
monkeypatch.setattr(
- b, "_restart_server_on_cpu_backend", lambda state, text, cancel: _Server(new_dir, False)
+ b,
+ "_restart_server_on_cpu_backend",
+ lambda state, text, cancel, **kw: _Server(new_dir, False),
)
+ # Published like a real load: a restart hands over only while _state still holds the server it replaces.
+ b._state = _State(family = detect_family("flux.1"), server = _Server(old_dir, fail_first))
b._generate_server(
- _State(family = detect_family("flux.1"), server = _Server(old_dir, fail_first)),
+ b._state,
prompt = "p",
negative_prompt = None,
width = 64,
@@ -2973,6 +2980,501 @@ def test_the_native_progress_total_is_sized_where_the_assets_are_fetched(monkeyp
assert b._loading.expected_bytes == 2_000
+def _png_bytes():
+ import io
+
+ from PIL import Image
+
+ buf = io.BytesIO()
+ Image.new("RGB", (8, 8)).save(buf, format = "PNG")
+ return buf.getvalue()
+
+
[email protected]("losses, recovers", [(1, True), (2, False)])
+def test_a_lost_vulkan_device_restarts_the_resident_server(monkeypatch, losses, recovers):
+ # The resident server stays up on a dead device after VK_ERROR_DEVICE_LOST: start the same build again
+ # (a fresh device) and retry the job once; a second loss in a row surfaces where no CPU rung applies (an Arc /
+ # ROCm host keeps Vulkan). Neither records a strike.
+ monkeypatch.setattr(bk, "_vulkan_cpu_rung_applies", lambda: False)
+ events: list = []
+ remaining = {"losses": losses}
+
+ class _FakeServer:
+ def __init__(self, binary):
+ self.binary = binary
+ events.append(("new", binary))
+
+ def start(self, files, **kwargs):
+ events.append(("start", kwargs.get("offload")))
+
+ def stop(self):
+ events.append(("stop", self.binary))
+
+ def img_gen(self, payload, **_kw):
+ if remaining["losses"]:
+ remaining["losses"] -= 1
+ raise RuntimeError("sd-server job failed: vk::Queue::submit: ErrorDeviceLost")
+ return [_png_bytes()]
+
+ b = _loaded_backend()
+ b._state = _pinned_state(b, policy_flags = ("--offload-to-cpu",), device = "vulkan")
+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server-vulkan"))
+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)
+ monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/bin/sd-server-cpu")
+ bk._accelerator_runtime_failures.clear()
+ kwargs = dict(
+ prompt = "a fox",
+ negative_prompt = None,
+ width = 64,
+ height = 64,
+ steps = 2,
+ seed = 1,
+ batch_size = 1,
+ cfg_scale = None,
+ flux_guidance = None,
+ lora_resolved = [],
+ cancel = threading.Event(),
+ )
+ if recovers:
+ images, _seeds = b._generate_server(b._state, **kwargs)
+ assert len(images) == 1
+ else:
+ with pytest.raises(RuntimeError, match = "ErrorDeviceLost"):
+ b._generate_server(b._state, **kwargs)
+ # The same build, with the load's own flags (device pin included), not the CPU backend.
+ assert ("new", "/bin/sd-server-vulkan") in events[1:]
+ assert ("start", b._state.offload_flags) in events or any(
+ e[0] == "start" and "--offload-to-cpu" in (e[1] or ()) for e in events
+ )
+ assert not bk._accelerator_runtime_failures
+
+
+def _repeated_device_loss_backend(
+ monkeypatch,
+ *,
+ losses,
+ gpu_explicit = False,
+):
+ """A CPU-torch Vulkan load whose first ``losses`` jobs lose the device, on whichever server runs them."""
+ events: list = []
+ remaining = {"losses": losses}
+ persisted: list = []
+
+ class _FakeServer:
+ lora_dir = None
+
+ def __init__(self, binary):
+ self.binary = binary
+
+ def start(self, files, **kwargs):
+ events.append(("start", self.binary, tuple(kwargs.get("extra_args") or ())))
+
+ def stop(self):
+ events.append(("stop", self.binary))
+
+ def is_alive(self):
+ return True
+
+ def img_gen(self, payload, **_kw):
+ if remaining["losses"]:
+ remaining["losses"] -= 1
+ raise RuntimeError("sd-server job failed: vk::Queue::submit: ErrorDeviceLost")
+ return [_png_bytes()]
+
+ b = _loaded_backend()
+ b._state = _pinned_state(b, policy_flags = ("--offload-to-cpu",), device = "vulkan")
+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server-vulkan"))
+ object.__setattr__(b._state, "mode", "server")
+ object.__setattr__(b._state, "gpu_explicit", gpu_explicit)
+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)
+ monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/bin/sd-server-vulkan")
+ monkeypatch.setattr(bk, "_vulkan_cpu_rung_applies", lambda: True)
+ monkeypatch.setattr(
+ bk, "fallback_accelerator_for", lambda acc: "cpu" if acc == "vulkan" else None
+ )
+ monkeypatch.setattr(
+ bk, "_persist_accelerator_runtime_failures", lambda records: persisted.append(records)
+ )
+ monkeypatch.setattr(bk, "_stored_accelerator_runtime_failures", lambda: {})
+ bk._accelerator_runtime_failures.clear()
+ return b, events, persisted
+
+
+def test_a_vulkan_device_lost_again_after_the_respawn_renders_this_load_on_cpu(monkeypatch):
+ # A host whose driver resets the card on every render (a Windows TDR on a slow iGPU) must not fail every request:
+ # the respawned server losing the device too moves this load alone to the CPU backend. Nothing is persisted, so
+ # the next load resolves the Vulkan build again.
+ b, events, persisted = _repeated_device_loss_backend(monkeypatch, losses = 2)
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1
+ starts = [e for e in events if e[0] == "start"]
+ # First the same-build respawn on Vulkan, then the CPU backend.
+ assert [s[2] for s in starts] == [(), ("--backend", "cpu")]
+ assert b._cpu_backend_forced is True
+ # The published state follows the server to CPU: status() no longer reports a Vulkan renderer, and the flags are
+ # the ones the CPU server got (the Vulkan device pin dropped).
+ assert b._state.device == "cpu"
+ assert b._state.physical_gpu_id is None
+ assert "--backend" not in b._state.offload_flags
+ assert b._state.offload_flags == ("--offload-to-cpu",)
+ assert b.status()["device"] == "cpu"
+ assert not persisted and not bk._accelerator_runtime_failures
+ # The failure text is not a strike either, and the next load still prefers Vulkan.
+ bk.note_accelerator_failure_from_output(
+ "/bin/sd-server-vulkan", "sd-server job failed: vk::Queue::submit: ErrorDeviceLost"
+ )
+ assert not persisted and not bk._accelerator_runtime_failures
+ assert bk.preferred_accelerator("vulkan") == "vulkan"
+ # Later requests on this load keep the CPU server without another restart.
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1
+ assert len([e for e in events if e[0] == "start"]) == 2
+
+
+def test_a_single_vulkan_device_loss_stays_on_vulkan(monkeypatch):
+ b, events, persisted = _repeated_device_loss_backend(monkeypatch, losses = 1)
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1
+ assert [e[2] for e in events if e[0] == "start"] == [()]
+ assert b._cpu_backend_forced is False
+ assert b._state.device == "vulkan"
+ assert not persisted
+
+
[email protected]("why", ["explicit_gpu", "no_cpu_rung", "fallback_disabled"])
+def test_a_repeated_vulkan_device_loss_without_a_cpu_rung_errors_with_a_hint(monkeypatch, why):
+ # A user-picked GPU is never moved to CPU behind their back, and an Arc / ROCm host (no CPU rung) or a disabled
+ # Vulkan fallback keeps Vulkan: the second loss surfaces, with what to try.
+ b, events, persisted = _repeated_device_loss_backend(
+ monkeypatch, losses = 2, gpu_explicit = why == "explicit_gpu"
+ )
+ if why == "no_cpu_rung":
+ monkeypatch.setattr(bk, "_vulkan_cpu_rung_applies", lambda: False)
+ if why == "fallback_disabled":
+ monkeypatch.setattr(bk, "fallback_accelerator_for", lambda acc: None)
+ with pytest.raises(RuntimeError, match = "ErrorDeviceLost") as info:
+ b._generate_server(b._state, **_gen_kwargs())
+ assert "twice in a row" in str(info.value)
+ assert not bk.output_shows_accelerator_failure(str(info.value))
+ assert all(e[2] != ("--backend", "cpu") for e in events if e[0] == "start")
+ assert b._cpu_backend_forced is False
+ assert not persisted
+
+
[email protected]("losses, device", [(1, "vulkan"), (2, "cpu")])
+def test_a_device_loss_replacement_restages_the_hires_upscaler(
+ monkeypatch, tmp_path, losses, device
+):
+ # Every replacement server (the same-build respawn after one lost device, the CPU backend after a second) has its
+ # own scratch dir, and the old server's stop removed the upscaler staged in its own: the retry must stage the model
+ # upscaler again under the server that runs it, or sd.cpp cannot find the hires upscaler file.
+ from core.inference import diffusion_hires as hires
+
+ b, events, persisted = _repeated_device_loss_backend(monkeypatch, losses = losses)
+ server_cls = type(b._state.server)
+ dirs: dict = {}
+ seen: list = []
+
+ def _lora_dir(self):
+ if id(self) not in dirs:
+ dirs[id(self)] = tmp_path / f"srv{len(dirs)}"
+ dirs[id(self)].mkdir()
+ return str(dirs[id(self)])
+
+ real_img_gen = server_cls.img_gen
+
+ def _img_gen(self, payload, **kw):
+ seen.append((self.lora_dir, dict(payload["hires"])))
+ return real_img_gen(self, payload, **kw)
+
+ monkeypatch.setattr(server_cls, "lora_dir", property(_lora_dir), raising = False)
+ monkeypatch.setattr(server_cls, "img_gen", _img_gen)
+ staged: list = []
+
+ def _stage(
+ upscaler_id,
+ scratch_dir,
+ *,
+ cancel = None,
+ ):
+ dest = Path(scratch_dir) / "hires_x.pth"
+ dest.write_bytes(b"w")
+ staged.append(dest)
+ return "hires_x", dest
+
+ monkeypatch.setattr(hires, "stage_sd_cpp_upscaler", _stage)
+ plan = hires.SdCppHiresPlan(
+ params = hires.HiresParams(upscaler = "realesrgan_x4plus"),
+ base_width = 64,
+ base_height = 64,
+ target = (96, 96),
+ refine_steps = 2,
+ )
+ images, _seeds = b._generate_server(b._state, hires_plan = plan, **_gen_kwargs())
+ assert len(images) == 1
+ assert b._state.device == device
+ assert not persisted
+ # One server per attempt, each sent the upscaler staged in its own dir.
+ assert len(seen) == losses + 1
+ assert len({d for d, _ in seen}) == losses + 1
+ assert all(h["upscaler"] == "hires_x" for _, h in seen)
+ assert [str(p.parent) for p in staged] == [d for d, _ in seen]
+ assert b._state.server.lora_dir == seen[-1][0]
+ assert not any(p.exists() for p in staged)
+
+
+def test_an_explicit_gpu_pool_marks_the_native_load_explicit(monkeypatch):
+ seen: dict = {}
+ monkeypatch.setattr(
+ "core.inference.diffusion_engine_router.off_torch_sd_cpp_device", lambda *a, **k: None
+ )
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ monkeypatch.setattr(b, "_run_load", lambda **kw: seen.update(kw))
+ monkeypatch.setattr(bk, "resolve_selected_cuda_ordinal", lambda ids: None, raising = False)
+ b.begin_load(
+ repo_id = "unsloth/Z-Image-Turbo-GGUF", gguf_filename = "z-image-turbo-Q4_K_M.gguf", gpu_ids = [0]
+ )
+ deadline = time.monotonic() + 5
+ while "gpu_explicit" not in seen and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert seen.get("gpu_explicit") is True
+
+
+def _device_loss_backend(
+ monkeypatch,
+ tmp_path,
+ *,
+ on_start = None,
+):
+ """A loaded backend whose first job loses its Vulkan device; each server gets its own lora dir."""
+ servers: list = []
+ payloads: list = []
+
+ class _FakeServer:
+ def __init__(self, binary):
+ self.binary = binary
+ self.lora_dir = str(tmp_path / f"srv{len(servers)}")
+ os.makedirs(self.lora_dir, exist_ok = True)
+ self.stopped = False
+ servers.append(self)
+
+ def start(self, files, **kwargs):
+ if on_start is not None:
+ on_start(self)
+
+ def stop(self):
+ self.stopped = True
+ shutil.rmtree(self.lora_dir, ignore_errors = True)
+
+ def is_alive(self):
+ return not self.stopped
+
+ def img_gen(self, payload, **_kw):
+ payloads.append((self, dict(payload)))
+ if len(payloads) == 1:
+ raise RuntimeError("sd-server job failed: vk::Queue::submit: ErrorDeviceLost")
+ return [_png_bytes()]
+
+ b = _loaded_backend()
+ b._state = _pinned_state(b, policy_flags = (), device = "vulkan")
+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server-vulkan"))
+ object.__setattr__(b._state, "mode", "server")
+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)
+ bk._accelerator_runtime_failures.clear()
+ return b, servers, payloads
+
+
+def _gen_kwargs(lora_resolved = ()):
+ return dict(
+ prompt = "a fox",
+ negative_prompt = None,
+ width = 64,
+ height = 64,
+ steps = 2,
+ seed = 1,
+ batch_size = 1,
+ cfg_scale = None,
+ flux_guidance = None,
+ lora_resolved = list(lora_resolved),
+ cancel = threading.Event(),
+ )
+
+
+def test_a_device_loss_retry_restages_the_loras_under_the_new_server(monkeypatch, tmp_path):
+ # The old server's stop removes its scratch dir and the staged adapters: the retry must stage them again.
+ from core.inference import diffusion_lora
+
+ def _materialize(
+ resolved,
+ stage,
+ real_files = False,
+ ):
+ os.makedirs(stage, exist_ok = True)
+ (Path(stage) / "adapter.gguf").write_bytes(b"x")
+ return [types.SimpleNamespace(path = str(Path(stage) / "adapter.gguf"), weight = 0.8)]
+
+ monkeypatch.setattr(diffusion_lora, "materialize_native_dir", _materialize)
+ b, servers, payloads = _device_loss_backend(monkeypatch, tmp_path)
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs(lora_resolved = ["adapter"]))
+ assert len(images) == 1 and len(servers) == 2
+ retry_server, retry_payload = payloads[-1]
+ assert retry_server is servers[1]
+ (entry,) = retry_payload["lora"]
+ # Staged under the replacement's own lora dir (the file exists there when the retry runs).
+ assert entry["path"] != payloads[0][1]["lora"][0]["path"]
+ assert entry["multiplier"] == 0.8
+
+
+def test_an_unload_during_the_respawn_leaves_no_live_server(monkeypatch, tmp_path):
+ # unload() racing the restart must find the replacement (pending) or the handover must stop it.
+ holder: dict = {}
+
+ def _unload_mid_start(server):
+ holder["backend"].unload()
+
+ b, servers, _payloads = _device_loss_backend(monkeypatch, tmp_path, on_start = _unload_mid_start)
+ holder["backend"] = b
+ monkeypatch.setattr(b, "status", lambda: {})
+ with pytest.raises(SdCppCancelled):
+ b._generate_server(b._state, **_gen_kwargs())
+ assert len(servers) == 2
+ assert all(s.stopped for s in servers)
+ assert b._state is None and b._pending_server is None
+
+
+def test_a_status_poll_during_the_respawn_keeps_the_replacement(monkeypatch, tmp_path):
+ # The UI polls /images/status while the respawn runs. The old server is already stopped then, and the idle-crash
+ # reap used to clear _state, so the handover read an unload and killed the healthy replacement.
+ holder: dict = {}
+ polls: list = []
+
+ def _poll_mid_start(server):
+ polls.append(holder["backend"].status()["loaded"])
+
+ b, servers, _payloads = _device_loss_backend(monkeypatch, tmp_path, on_start = _poll_mid_start)
+ holder["backend"] = b
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1 and len(servers) == 2
+ assert polls == [True]
+ assert b._state is not None and b._state.server is servers[1]
+ assert not servers[1].stopped and b._replacing_server is None
+ # Once the generation is done, a server that really died is reaped again.
+ servers[1].stopped = True
+ assert b.status()["loaded"] is False
+
+
+def test_a_status_poll_during_the_cpu_backend_restart_keeps_the_load(monkeypatch):
+ import dataclasses
+
+ class _FakeServer:
+ def __init__(self, binary):
+ self.binary = binary
+ self.stopped = False
+
+ def start(self, files, **kwargs):
+ polls.append(b.status()["loaded"])
+
+ def stop(self):
+ self.stopped = True
+
+ def is_alive(self):
+ return not self.stopped
+
+ polls: list = []
+ b = _loaded_backend()
+ b._state = _pinned_state(b, policy_flags = ("--offload-to-cpu",))
+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server"))
+ object.__setattr__(b._state, "mode", "server")
+ monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/bin/sd-server")
+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)
+ state = b._state
+ server = b._restart_server_on_cpu_backend(
+ state,
+ "ggml_metal_op_encode_impl: unsupported op 'MUL_MAT' -> ggml_abort",
+ threading.Event(),
+ )
+ assert server is not None and polls == [True] and b._state is state
+ b._hand_over_replacement(dataclasses.replace(state, server = server), server, state.server)
+ assert b._state.server is server and not server.stopped
+
+
+def test_a_superseding_load_during_the_respawn_is_not_overwritten(monkeypatch, tmp_path):
+ # A newer load that commits while the replacement starts owns _state: the handover must not publish the old
+ # model's replacement over it, and the replacement must not be left running untracked.
+ import dataclasses
+
+ holder: dict = {}
+
+ def _supersede_mid_start(server):
+ b = holder["backend"]
+ newer = _FakeNewer("/bin/sd-server-newer")
+ holder["newer"] = newer
+ with b._lock:
+ b._state = dataclasses.replace(b._state, server = newer)
+
+ class _FakeNewer:
+ def __init__(self, binary):
+ self.binary = binary
+ self.stopped = False
+
+ def stop(self):
+ self.stopped = True
+
+ def is_alive(self):
+ return not self.stopped
+
+ b, servers, payloads = _device_loss_backend(
+ monkeypatch, tmp_path, on_start = _supersede_mid_start
+ )
+ holder["backend"] = b
+ monkeypatch.setattr(b, "status", lambda: {})
+ with pytest.raises(SdCppCancelled):
+ b._generate_server(b._state, **_gen_kwargs())
+ assert len(servers) == 2 and servers[1].stopped # the stale replacement is stopped
+ assert b._state.server is holder["newer"] and not holder["newer"].stopped
+ assert len(payloads) == 1 and b._pending_server is None
+
+
+def test_a_cancel_during_the_respawn_stops_the_replacement_start(monkeypatch, tmp_path):
+ # cancel_generate() (or a superseding load) only sets the generation's event; the replacement start blocks for the
+ # model load while _generate_lock is held, so it must notice and bail instead of loading for the full timeout.
+ kwargs = _gen_kwargs()
+ cancel = kwargs["cancel"]
+
+ def _cancel_mid_start(server):
+ cancel.set()
+ deadline = time.monotonic() + 5.0
+ while time.monotonic() < deadline:
+ if server.stopped:
+ raise RuntimeError("sd-server startup aborted before ready")
+ time.sleep(0.01)
+
+ b, servers, payloads = _device_loss_backend(monkeypatch, tmp_path, on_start = _cancel_mid_start)
+ monkeypatch.setattr(b, "status", lambda: {})
+ started = time.monotonic()
+ with pytest.raises(SdCppCancelled):
+ b._generate_server(b._state, **kwargs)
+ assert time.monotonic() - started < 4.0
+ assert len(servers) == 2 and all(s.stopped for s in servers)
+ assert len(payloads) == 1 # no retry job was posted
+ assert b._pending_server is None and b._replacing_server is None
+
+
+def test_an_unload_before_the_respawn_starts_skips_it(monkeypatch, tmp_path):
+ # unload() wins outright when it lands first: no replacement is started at all.
+ b, servers, _payloads = _device_loss_backend(monkeypatch, tmp_path)
+ state = b._state
+ monkeypatch.setattr(b, "status", lambda: {})
+ b.unload()
+ assert (
+ b._respawn_server_after_device_loss(
+ state, "vk::Queue::submit: ErrorDeviceLost", threading.Event()
+ )
+ is None
+ )
+ assert len(servers) == 1 and b._replacing_server is None
+
+
@pytest.mark.parametrize(
"device,ccs,expected",
[
diff --git a/studio/backend/tests/test_sd_cpp_vulkan_cpu_torch.py b/studio/backend/tests/test_sd_cpp_vulkan_cpu_torch.py
index 6b411607f28..ac200bcb42e 100644
--- a/studio/backend/tests/test_sd_cpp_vulkan_cpu_torch.py
+++ b/studio/backend/tests/test_sd_cpp_vulkan_cpu_torch.py
@@ -70,6 +70,7 @@ def _pinned_host(monkeypatch):
monkeypatch.setattr(amd, "the_vulkan_loader_has_no_usable_driver", lambda: False)
monkeypatch.setattr(r, "_hardware_vulkan_driver", lambda vendors: True)
monkeypatch.setattr(r, "_nvidia_adapter_present", lambda: False)
+ monkeypatch.setattr(r, "_reachable_intel_arc_render_node", lambda: True, raising = False)
@pytest.mark.parametrize(
@@ -77,8 +78,8 @@ def _pinned_host(monkeypatch):
[
((_VEGA,), "vulkan"),
((_ARC_LINUX,), "vulkan"),
- # Unpinned sd.cpp could take the UHD iGPU first.
- ((_UHD_LINUX, _VEGA), "auto"),
+ # Unpinned sd.cpp takes a discrete GPU before an iGPU: the card renders, not the UHD iGPU.
+ ((_UHD_LINUX, _VEGA), "vulkan"),
((_ARC_LINUX, _VEGA), "vulkan"),
((_UHD_LINUX,), "auto"),
((), "auto"),
@@ -143,9 +144,43 @@ def test_an_unmounted_uhd_node_does_not_block_the_amd_card(monkeypatch):
monkeypatch.setattr(r, "_physical_inventory", lambda: _host(_UHD_LINUX, _VEGA))
monkeypatch.setattr(r, "_accessible_render_vendors", lambda vendors: {"amd"} & set(vendors))
assert r.image_install_accelerator("cpu") == "vulkan"
- # Both nodes mounted: the UHD iGPU could be taken first.
+ # Both nodes mounted: the dedicated card is still the one sd.cpp takes.
monkeypatch.setattr(r, "_accessible_render_vendors", lambda vendors: set(vendors))
- assert r.image_install_accelerator("cpu") == "auto"
+ assert r.image_install_accelerator("cpu") == "vulkan"
+
+
[email protected]("arc_node_reachable, expected", [(True, "vulkan"), (False, "auto")])
+def test_a_uhd_beside_an_arc_needs_the_arcs_own_node(monkeypatch, arc_node_reachable, expected):
+ # The vendor-level node check passes on the iGPU's node alone: a container mounting only it sees the UHD.
+ monkeypatch.setattr(r, "_physical_inventory", lambda: _host(_UHD_LINUX, _ARC_LINUX))
+ monkeypatch.setattr(
+ r, "_reachable_intel_arc_render_node", lambda: arc_node_reachable, raising = False
+ )
+ assert r.image_install_accelerator("cpu") == expected
+
+
+ "arc_node_reachable, drivers, expected",
+ [
+ # The Arc node is not mounted and only the Intel driver loads: Intel Vulkan sees the UHD alone and the AMD
+ # card has no driver, so the iGPU would render.
+ (False, {"intel"}, "auto"),
+ # The AMD card's own driver loads: it renders, whatever the Arc's node.
+ (False, {"amd"}, "vulkan"),
+ (False, {"amd", "intel"}, "vulkan"),
+ # The Arc's node is reachable: the Intel driver drives the Arc, a discrete GPU sd.cpp takes first.
+ (True, {"intel"}, "vulkan"),
+ ],
+)
+def test_an_arc_beside_an_amd_card_and_a_uhd_still_needs_its_own_node(
+ monkeypatch, arc_node_reachable, drivers, expected
+):
+ monkeypatch.setattr(r, "_physical_inventory", lambda: _host(_VEGA, _ARC_LINUX, _UHD_LINUX))
+ monkeypatch.setattr(
+ r, "_reachable_intel_arc_render_node", lambda: arc_node_reachable, raising = False
+ )
+ monkeypatch.setattr(r, "_hardware_vulkan_driver", lambda vendors: bool(set(vendors) & drivers))
+ assert r.image_install_accelerator("cpu") == expected
@pytest.mark.parametrize("value", ["", "default"])
@@ -164,15 +199,20 @@ def test_a_malformed_ggml_vulkan_mask_keeps_cpu(monkeypatch, value):
@pytest.mark.parametrize(
"output",
[
- "vk::Queue::submit: ErrorDeviceLost",
- "VK_ERROR_DEVICE_LOST",
"vk::createInstance: ErrorIncompatibleDriver",
+ "vk::createInstance: ErrorInitializationFailed",
],
)
def test_vulkan_failures_are_recognised(output):
assert sd_cpp_backend.output_shows_accelerator_failure(output)
[email protected]("output", ["vk::Queue::submit: ErrorDeviceLost", "VK_ERROR_DEVICE_LOST"])
+def test_a_lost_vulkan_device_is_not_a_build_failure(output):
+ # Recoverable by recreating the device (a TDR, a scheduler timeout): no evidence about the build.
+ assert not sd_cpp_backend.output_shows_accelerator_failure(output)
+
+
def test_an_incompatible_vulkan_driver_is_decisive():
assert sd_cpp_backend.output_shows_decisive_accelerator_failure("ErrorIncompatibleDriver")
assert not sd_cpp_backend.output_shows_decisive_accelerator_failure("ErrorDeviceLost")
@@ -401,6 +441,16 @@ def test_the_render_node_and_the_driver_must_share_a_vendor(monkeypatch):
assert r.image_install_accelerator("cpu") == "vulkan"
+def test_a_uhd_beside_an_amd_card_needs_the_cards_own_driver(monkeypatch):
+ # Both nodes mounted, but only an Intel ICD loads: Vulkan then enumerates the iGPU alone and the AMD card is not a
+ # Vulkan device at all, so the UHD would render.
+ monkeypatch.setattr(r, "_physical_inventory", lambda: _host(_UHD_LINUX, _VEGA))
+ monkeypatch.setattr(r, "_hardware_vulkan_driver", lambda vendors: "intel" in vendors)
+ assert r.image_install_accelerator("cpu") == "auto"
+ monkeypatch.setattr(r, "_hardware_vulkan_driver", lambda vendors: "amd" in vendors)
+ assert r.image_install_accelerator("cpu") == "vulkan"
+
+
@pytest.mark.parametrize(
"records,live,expected",
[
@@ -433,3 +483,176 @@ def test_windows_nvidia_without_nvidia_smi_is_found_in_the_adapter_registry(
monkeypatch.setattr(hw, "_windows_live_adapter_names", lambda: live)
assert r._nvidia_adapter_present() is expected
assert seen == [0x10DE]
+
+
+def _empty_failure_records(monkeypatch, stored = None):
+ monkeypatch.setattr(sd_cpp_backend, "_accelerator_runtime_failures", {})
+ monkeypatch.setattr(
+ sd_cpp_backend, "_stored_accelerator_runtime_failures", lambda: dict(stored or {})
+ )
+ monkeypatch.setattr(sd_cpp_backend, "_persist_accelerator_runtime_failures", lambda _r: None)
+ monkeypatch.setattr(sd_cpp_backend, "_accelerator_fingerprint", lambda *_a: {"bundle": "t"})
+
+
[email protected]("cpu_torch", [True, False])
+def test_two_lost_vulkan_devices_keep_the_vulkan_build(monkeypatch, cpu_torch):
+ """An Arc / ROCm-on-Vulkan host (and a CPU-torch one) that lost the device twice stays on the GPU build."""
+ _empty_failure_records(monkeypatch)
+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: cpu_torch)
+ monkeypatch.setattr(sd_cpp_backend, "_installed_accelerator_of", lambda _b: "vulkan")
+ for _ in range(2):
+ sd_cpp_backend.note_accelerator_failure_from_output(
+ "/x/sd-cli", "vk::Queue::submit: ErrorDeviceLost"
+ )
+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"
+
+
+def test_ambiguous_vulkan_strikes_move_only_a_cpu_torch_host(monkeypatch):
+ """Two ambiguous failures: the CPU-torch route falls to CPU sd.cpp, an XPU / ROCm host keeps Vulkan."""
+ _empty_failure_records(monkeypatch)
+ for _ in range(2):
+ sd_cpp_backend.note_accelerator_runtime_failure("vulkan", proven = False)
+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: True)
+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "cpu"
+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: False)
+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"
+ # Decisive evidence still moves every host: a build that cannot run must not leave it with nothing.
+ sd_cpp_backend.note_accelerator_runtime_failure("vulkan", proven = True)
+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "cpu"
+
+
+def test_a_vulkan_record_from_before_device_lost_stopped_counting_is_ignored(monkeypatch):
+ """Two ambiguous strikes persisted under the old rule may be two lost devices: they no longer divert."""
+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: True)
+ old = {"vulkan": {"strikes": 2, "proven": False, "fingerprint": {"bundle": "t"}}}
+ _empty_failure_records(monkeypatch, stored = old)
+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"
+ # A decisive one written then still holds.
+ proven = {"vulkan": {"strikes": 1, "proven": True, "fingerprint": {"bundle": "t"}}}
+ _empty_failure_records(monkeypatch, stored = proven)
+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "cpu"
+
+
+def test_an_old_rule_record_keeps_only_the_proven_cards_known():
+ """Card A proven, card B ambiguous only, no unscoped proof: B must not divert on A's failure."""
+ old = {
+ "strikes": 2,
+ "proven": True,
+ "fingerprint": {"bundle": "t"},
+ "cards": ["Card A", "Card B"],
+ "per_card": {
+ "Card A": {"strikes": 1, "proven": True},
+ "Card B": {"strikes": 1, "proven": False},
+ },
+ }
+ kept = sd_cpp_backend._decisive_evidence_only(old)
+ assert kept["cards"] == ["Card A"]
+ fp = {"bundle": "t"}
+ assert sd_cpp_backend._record_diverts(kept, fp, card = "Card A") is True
+ assert sd_cpp_backend._record_diverts(kept, fp, card = "Card B") is False
+
+
[email protected]("card,diverts", [("Card A", True), ("Card B", False), (None, True)])
+def test_an_old_rule_proof_without_tallies_keeps_its_cards(card, diverts):
+ """Proof naming cards but no per-card tallies diverted only those cards; migrating it must not widen it."""
+ old = {"strikes": 1, "proven": True, "fingerprint": {"bundle": "t"}, "cards": ["Card A"]}
+ fp = {"bundle": "t"}
+ assert sd_cpp_backend._record_diverts(old, fp, card = card) is diverts
+ kept = sd_cpp_backend._record_under_current_rule("vulkan", old)
+ assert sd_cpp_backend._record_diverts(kept, fp, card = card) is diverts
+
+
+def test_the_fallback_status_counts_the_same_evidence_as_the_runtime(monkeypatch):
+ """Settings must not report a diversion the runtime decision no longer makes."""
+ monkeypatch.setattr(sd_cpp_backend, "sd_cpp_vulkan_fallback_enabled", lambda: True)
+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: True)
+ old = {"vulkan": {"strikes": 2, "proven": False, "fingerprint": {"bundle": "t"}}}
+ _empty_failure_records(monkeypatch, stored = old)
+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"
+ assert sd_cpp_backend.accelerator_runtime_failure_state()["diverting"] is False
+ # Current ambiguous strikes off the CPU-torch route: kept on Vulkan, and reported so.
+ _empty_failure_records(monkeypatch)
+ for _ in range(2):
+ sd_cpp_backend.note_accelerator_runtime_failure("vulkan", proven = False)
+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: False)
+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"
+ assert sd_cpp_backend.accelerator_runtime_failure_state()["diverting"] is False
+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: True)
+ assert sd_cpp_backend.accelerator_runtime_failure_state()["diverting"] is True
+
+
[email protected]("value", ["0 1", "0, 1", "1 "])
+def test_a_space_separated_ggml_vulkan_mask_is_a_mask(monkeypatch, value):
+ # ggml reads commas as spaces.
+ monkeypatch.setenv("GGML_VK_VISIBLE_DEVICES", value)
+ assert r.image_install_accelerator("cpu") == "vulkan"
+
+
+_LISTING = (
+ "CPU\tAMD Ryzen 9\nVulkan0\tAMD Radeon RX 7800 XT (RADV NAVI32)\nROCm0\tAMD Radeon RX 7800 XT\n"
+)
+
+
+def test_device_listing_readers_share_one_probe(monkeypatch, tmp_path):
+ binary = tmp_path / "sd-cli"
+ binary.write_bytes(b"x")
+ calls: list = []
+
+ def _probe(_binary, *args):
+ calls.append(args)
+ return _LISTING
+
+ monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})
+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", _probe)
+ assert sd_cpp_backend.sd_cpp_device_name_for_ordinal(str(binary), 0) == "ROCm0"
+ sd_cpp_backend.sd_cpp_device_named(str(binary), "AMD Radeon RX 7800 XT")
+ assert sd_cpp_backend.sd_cpp_listed_device(str(binary))[0] == "vulkan"
+ assert calls == [("--list-devices",)]
+ # A different device selection is a different listing.
+ monkeypatch.setenv("GGML_VK_VISIBLE_DEVICES", "0")
+ sd_cpp_backend.sd_cpp_listed_device(str(binary))
+ assert len(calls) == 2
+
+
+def test_a_hardware_inventory_change_reprobes_the_listing(monkeypatch, tmp_path):
+ # Same file, same env, but the eGPU was unplugged (or the driver reset): the old listing names a device that is gone.
+ from utils.hardware import gpu_query, hardware
+
+ binary = tmp_path / "sd-cli"
+ binary.write_bytes(b"x")
+ calls: list = []
+
+ def _probe(_binary, *args):
+ calls.append(args)
+ return _LISTING
+
+ monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})
+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", _probe)
+ sd_cpp_backend.sd_cpp_listed_device(str(binary))
+ sd_cpp_backend.sd_cpp_listed_device(str(binary))
+ assert len(calls) == 1
+ # An explicit re-detection (hotplug, driver reset).
+ gpu_query.invalidate_static("test hotplug")
+ sd_cpp_backend.sd_cpp_listed_device(str(binary))
+ assert len(calls) == 2
+ # A periodic inventory refresh that found a different device set.
+ before = [{"vendor": "amd", "name": "AMD Radeon RX 7800 XT"}]
+ monkeypatch.setattr(hardware, "_physical_gpu_inventory_cache", (0.0, {"devices": before}))
+ monkeypatch.setattr(hardware, "_probe_physical_gpu_inventory", lambda: {"devices": []})
+ hardware._run_physical_gpu_inventory_probe()
+ sd_cpp_backend.sd_cpp_listed_device(str(binary))
+ assert len(calls) == 3
+ # A refresh that found the same devices keeps the listing.
+ hardware._run_physical_gpu_inventory_probe()
+ sd_cpp_backend.sd_cpp_listed_device(str(binary))
+ assert len(calls) == 3
+
+
+def test_an_unreadable_listing_is_not_cached(monkeypatch, tmp_path):
+ binary = tmp_path / "sd-cli"
+ binary.write_bytes(b"x")
+ answers = [None, _LISTING]
+ monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})
+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", lambda *_a: answers.pop(0))
+ assert sd_cpp_backend.sd_cpp_device_name_for_ordinal(str(binary), 0) is None
+ assert sd_cpp_backend.sd_cpp_device_name_for_ordinal(str(binary), 0) == "ROCm0"
diff --git a/studio/backend/utils/hardware/gpu_query.py b/studio/backend/utils/hardware/gpu_query.py
index 32a1575c971..87fe66a9c9f 100644
--- a/studio/backend/utils/hardware/gpu_query.py
+++ b/studio/backend/utils/hardware/gpu_query.py
@@ -206,6 +206,11 @@ def invalidate_static(reason: str = "") -> None:
logger.debug("GPU inventory readings invalidated: %s", reason)
+def static_generation() -> int:
+ """Bumped by invalidate_static() and reset(): an inventory read before the bump may describe a different host."""
+ return _static_gen
+
+
def reset() -> None:
global _static_gen, _slow_until, _stats, _reset_epoch
_events.invalidate_gpu_memory("reset")
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 7a1cd2f9d1b..93b2650d0e0 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -288,6 +288,8 @@ _physical_gpu_inventory_refreshing = False
# One re-detection request per recovery, or a poll every few seconds starves detection.
_REDETECTION_REQUESTED = False
_physical_gpu_inventory_cache: Optional[tuple[float, Dict[str, Any]]] = None
+# Bumped when a refresh finds a different device set, so device-listing caches can tell a stale answer apart.
+_physical_gpu_inventory_generation = 0
# Same treatment for the torch side: classify_torch_build() calls availability probes that block on a wedged driver, and /api/health and /api/liveness reach it through the verdict.
_TORCH_BUILD_SNAPSHOT_TTL_SECONDS = 60.0
_torch_build_snapshot_lock = threading.Lock()
@@ -753,7 +755,7 @@ def _carry_unanswered_vendors_forward(
def _run_physical_gpu_inventory_probe() -> Dict[str, Any]:
"""One probe pass, stored in the cache. Never raises."""
- global _physical_gpu_inventory_cache
+ global _physical_gpu_inventory_cache, _physical_gpu_inventory_generation
previous = _physical_gpu_inventory_cache
try:
inventory = _probe_physical_gpu_inventory()
@@ -765,10 +767,19 @@ def _run_physical_gpu_inventory_probe() -> Dict[str, Any]:
inventory = _carry_unanswered_vendors_forward(
inventory, previous[1] if previous is not None else None
)
+ if previous is not None and previous[1].get("devices") != inventory.get("devices"):
+ _physical_gpu_inventory_generation += 1
_physical_gpu_inventory_cache = (time.monotonic(), inventory)
return inventory
+def physical_gpu_inventory_generation() -> int:
+ """Changes whenever the GPUs this host has may have changed: a refresh that found a different device set, or an
+ explicit re-detection (``gpu_query.invalidate_static``). Never probes."""
+ from . import gpu_query
+ return _physical_gpu_inventory_generation + gpu_query.static_generation()
+
+
def get_physical_gpu_inventory(*, block: bool = True) -> Dict[str, Any]:
"""GPUs the OS enumerates, whether or not PyTorch can use them: ``{available, devices, sources, unknown}``. Runs regardless of DEVICE and never raises. ``block=False`` for anything on a request path -- the NVIDIA half shells out with a 10s timeout behind a lock, so a hung driver would stall the event loop every TTL; a non-blocking caller takes the last answer, kicks a daemon-thread refresh, and gets the new value later."""
now = time.monotonic()