Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio sd.cpp: retry an image render that sd-server refused for VRAM under half the budget (#17339)

oobaboogaf2e923eb95a2
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index e0c9693d811..2df626dcdf5 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -8,7 +8,7 @@ from __future__ import annotations
import functools
import os
-from collections.abc import Sequence
+from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from typing import Optional
@@ -709,6 +709,60 @@ def is_device_oom_text(text: Optional[str]) -> bool:
return False
+# sd.cpp ``fail_segment`` capacity-check stages (ggml_runner.cpp); "workspace allocation" is an allocator failure.
+_SEGMENT_REFUSAL_MARKERS: tuple[str, ...] = (
+ "failed during weight preparation",
+ "failed during workspace capacity check",
+ "failed during allocated capacity check",
+)
+# Allocator failures on the host RAM that weight preparation stages into, as well as on the device.
+_ALLOCATION_FAILURE_MARKERS: tuple[str, ...] = (
+ *_HOST_OOM_MARKERS,
+ "erroroutofhostmemory",
+ "failed to allocate",
+ "backend buffer failed",
+)
+# sd.cpp fills free-512 MiB with cached weights, so each phase needs memory the last one released; half leaves it
+# unclaimed when the driver reports the release late (Windows may defer destroying allocations).
+IMAGE_VRAM_RETRY_SCALE = 0.5
+
+
+def is_vram_refusal_text(text: Optional[str]) -> bool:
+ """sd.cpp's model manager declined a segment for VRAM, and no allocation failed outright (that keeps its error)."""
+ lines = str(text or "").lower().splitlines()
+ if not any(marker in line for line in lines for marker in _SEGMENT_REFUSAL_MARKERS):
+ return False
+ return not any(
+ "failed during" not in line
+ and (
+ is_device_oom_text(line)
+ or any(marker in line for marker in _ALLOCATION_FAILURE_MARKERS)
+ )
+ for line in lines
+ )
+
+
+def image_vram_retry_budget(
+ planned_free_mib: Optional[Mapping[str, int]], render_devices: Iterable[str] = ()
+) -> Optional[str]:
+ """``--max-vram`` for the image retry: half of each planned card's free VRAM (``CUDA0=..,CUDA1=..`` for several),
+ skipping cards under the floor. None if none qualifies or a ``render_devices`` card does not."""
+ planned = dict(planned_free_mib or {})
+ budgets: dict[str, float] = {}
+ for name, mib in planned.items():
+ gib = graph_cut_budget_gib(int(mib) << 20, reserve_bytes = 0, scale = IMAGE_VRAM_RETRY_SCALE)
+ if gib is not None:
+ budgets[name] = gib
+ if not budgets:
+ return None
+ known = {name.lower() for name in budgets}
+ if any(str(device).lower() not in known for device in render_devices):
+ return None
+ if len(planned) == 1:
+ return _format_gib(next(iter(budgets.values())))
+ return ",".join(f"{name}={_format_gib(gib)}" for name, gib in budgets.items())
+
+
def device_backend_flags(
device_name: Optional[str],
offload: Optional[list[str]] = None,
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index d5434f7ac95..f1477ec8a80 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -2930,10 +2930,14 @@ class _SdState:
gpu_explicit: bool = False
multi_gpu: Optional[dict[str, Any]] = None
split_physical_gpu_id: Optional[int] = None
+ vram_budget: Optional[str] = None
def spawn_env(self) -> Optional[dict[str, str]]:
return dict(self.child_env) or None
+ def vram_budget_args(self) -> list[str]:
+ return ["--max-vram", self.vram_budget] if self.vram_budget else []
+
@property
def placed_ordinal(self) -> Optional[int]:
"""Torch ordinal of ``physical_gpu_id`` (a split's diffusion card); without it card-sharing helpers (upsampler) split over all GPUs."""
@@ -5027,6 +5031,7 @@ class SdCppDiffusionBackend:
# One same-build respawn per request; a second loss moves this load to CPU where the rung applies. Persists nothing.
device_respawned = False
device_losses = 0
+ vram_restarted = False
hires_payload: Optional[dict] = None
hires_staged: list[Path] = []
@@ -5099,10 +5104,12 @@ class SdCppDiffusionBackend:
except RuntimeError as first_exc:
# A ggml unsupported-op abort killed the server: this graph cannot run on the GPU backend at all,
# so restart the model on the CPU backend once and retry this chunk. A lost Vulkan device needs a
- # fresh device: respawn the same build once. Any other death propagates.
+ # fresh device: respawn the same build once. A VRAM refusal restarts once under a smaller
+ # budget. Any other death propagates.
exc = first_exc
while True:
moved_to_cpu = False
+ vram_budget: Optional[str] = None
server = self._restart_server_on_cpu_backend(state, str(exc), cancel)
if server is None and output_shows_vulkan_device_lost(str(exc)):
device_losses += 1
@@ -5120,10 +5127,17 @@ class SdCppDiffusionBackend:
state, str(exc), cancel
)
device_respawned = server is not None
+ if server is None and not vram_restarted:
+ server, vram_budget = self._restart_server_with_vram_headroom(
+ state, str(exc), cancel
+ )
+ vram_restarted = server is not None
if server is None:
raise exc
old_server = state.server
state = replace(state, server = server)
+ if vram_budget is not None:
+ state = replace(state, vram_budget = vram_budget)
if moved_to_cpu:
# Placement follows the server: the flags the CPU server got, no Vulkan pin.
state = replace(
@@ -5287,7 +5301,59 @@ class SdCppDiffusionBackend:
native_speed = state.native_speed,
threads = state.threads,
env = state.spawn_env(),
+ extra_args = state.vram_budget_args(),
+ )
+
+ def _restart_server_with_vram_headroom(
+ self, state: _SdState, error_text: str, cancel: threading.Event
+ ) -> tuple[Optional[SdCppServer], Optional[str]]:
+ """Restart sd-server under ``--max-vram`` after sd.cpp refused a render for VRAM; returns the server and its
+ budget, or ``(None, None)``. The budget comes from the first refusing server's plan and is reused: a server
+ restarted while another application held the card planned too little to halve, and keeps refusing."""
+ if not sd_cpp_args_mod.is_vram_refusal_text(error_text):
+ return None, None
+ if state.device == "cpu" or self._cpu_backend_forced:
+ return None, None
+ if state.server is None or cancel.is_set():
+ return None, None
+ binary = state.server.binary
+ # A two-card split already carries per-device budgets, and an older build rejects the flag.
+ if "--max-vram" in state.offload_flags or "--max-vram" not in sd_cpp_graph_cut_options(
+ binary
+ ):
+ return None, None
+ planned = dict(getattr(state.server, "planned_free_vram_mib", None) or {})
+ budget = state.vram_budget or sd_cpp_args_mod.image_vram_retry_budget(
+ planned, _backend_devices(state.offload_flags)
+ )
+ if budget is None or ("=" in budget and not _layer_split_supported(binary)):
+ return None, None
+ if not self._begin_replacement(state):
+ return None, None
+ logger.warning(
+ "sd-server refused the render for VRAM; restarting it with --max-vram %s (half the free VRAM this "
+ "load first planned with; the refusing server planned with, in MiB: %s) and retrying once. "
+ "Details: %s",
+ budget,
+ planned,
+ error_text[:300],
+ )
+ state.server.stop()
+ server = SdCppServer(binary)
+ with self._lock:
+ self._pending_server = server
+ started = self._start_replacement(
+ state,
+ server,
+ cancel,
+ vae_format = state.vae_format,
+ offload = state.offload_flags,
+ native_speed = state.native_speed,
+ threads = state.threads,
+ env = state.spawn_env(),
+ extra_args = ["--max-vram", budget],
)
+ return started, (budget if started is not None else None)
def _restart_server_on_cpu_backend(
self,
diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.py
index a8d3fdc7739..abfac256399 100644
--- a/studio/backend/core/inference/sd_cpp_server.py
+++ b/studio/backend/core/inference/sd_cpp_server.py
@@ -65,6 +65,11 @@ _TOTAL_PARAMS_VRAM_RE = re.compile(
re.IGNORECASE,
)
+# sd.cpp auto-fit device line "[I] CUDA0 <card> free N MiB, budget M MiB"; the RAM "params budget" line won't match.
+_AUTO_FIT_DEVICE_FREE_RE = re.compile(
+ r"^\[I\]\s+(\S+)\s.*\bfree\s+([0-9]+)\s*MiB,\s*budget\s+[0-9]+\s*MiB\b"
+)
+
_TERMINAL_OK = "completed"
_TERMINAL_FAIL = "failed"
_TERMINAL_CANCELLED = "cancelled"
@@ -165,6 +170,7 @@ class SdCppServer:
self.port: Optional[int] = None
self._process: Optional[subprocess.Popen] = None
self._resident_params_vram_gb: Optional[float] = None
+ self._planned_free_vram_mib: dict[str, int] = {}
# Bounded tail buffer shared by the drain thread (appends) and readers (diagnostics).
self._tail: deque[str] = deque(maxlen = 200)
self._lines_seen = 0
@@ -267,6 +273,7 @@ class SdCppServer:
# Clear in place; reassigning [] would drop the maxlen bound and grow unbounded.
self._tail.clear()
self._resident_params_vram_gb = None
+ self._planned_free_vram_mib = {}
self._spawn_error: Optional[Exception] = None
spawned = threading.Event()
@@ -468,6 +475,9 @@ class SdCppServer:
value = float(match.group(1)) / 1024.0
if value > 0:
self._resident_params_vram_gb = value
+ match = _AUTO_FIT_DEVICE_FREE_RE.search(line)
+ if match is not None:
+ self._planned_free_vram_mib[match.group(1)] = int(match.group(2))
logger.debug("[sd-server] %s", line)
cb = self._step_listener
if cb is not None:
@@ -486,6 +496,11 @@ class SdCppServer:
if wake is not None:
wake.set()
+ @property
+ def planned_free_vram_mib(self) -> dict[str, int]:
+ """Free VRAM per device (``{"CUDA0": 6666}``) from sd.cpp's startup auto-fit plan; empty without one."""
+ return dict(self._planned_free_vram_mib)
+
def reclaimable_params_vram_gb(
self, physical_gpu_id: Optional[int]
) -> Optional[dict[int, float]]:
diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.py
index 249918cc39d..d39b8ac33bd 100644
--- a/studio/backend/tests/test_sd_cpp_backend.py
+++ b/studio/backend/tests/test_sd_cpp_backend.py
@@ -3133,6 +3133,192 @@ def test_a_single_vulkan_device_loss_stays_on_vulkan(monkeypatch):
assert not persisted
+_VRAM_REFUSAL = (
+ "sd-server generation failed: generation_failed: generate_image returned no results\n"
+ "Last output:\n"
+ "[E] MOSS-Image segment 1/43 (moss_image.layers.0) failed during weight preparation --- ggml_runner.cpp:934\n"
+ "[E] diffusion model compute failed --- diffusion_engine.cpp:2729\n"
+ "[E] Diffusion model sampling failed --- diffusion_engine.cpp:2976\n"
+ "[E] sampling for image 1/1 failed after 0.09s --- image.cpp:1007"
+)
+
+
+_ALLOCATION_FAILURE = (
+ "sd-server generation failed: generation_failed: generate_image returned no results\n"
+ "Last output:\n"
+ "[E] ggml: ggml_backend_cuda_buffer_type_alloc_buffer: allocating 210.08 MiB on device 0: "
+ "cudaMalloc failed: out of memory\n"
+ "[E] ggml: alloc_tensor_range: failed to allocate CUDA0 buffer of size 220288512\n"
+ "[E] MOSS-Image segment 12/43 (moss_image.layers.11) failed during {stage} --- ggml_runner.cpp:934\n"
+ "[E] diffusion model compute failed --- diffusion_engine.cpp:2729"
+)
+
+
+def _vram_refusal_backend(
+ monkeypatch,
+ *,
+ refusals,
+ planned_free_mib = None,
+ restarted_plans = (),
+ error = _VRAM_REFUSAL,
+ max_vram_supported = True,
+ per_device_supported = True,
+ policy_flags = (),
+ device = "cuda",
+):
+ """CUDA load pinned to CUDA1 whose first ``refusals`` jobs fail with ``error``; ``restarted_plans`` are the
+ free-VRAM plans each replacement server reads in turn."""
+ events: list = []
+ remaining = {"refusals": refusals}
+ planned = {"CUDA1": 6666} if planned_free_mib is None else planned_free_mib
+ plans = [planned, *restarted_plans]
+
+ class _FakeServer:
+ lora_dir = None
+
+ def __init__(self, binary):
+ self.binary = binary
+ self.planned_free_vram_mib = dict(plans.pop(0) if len(plans) > 1 else plans[0])
+
+ def start(self, files, **kwargs):
+ events.append(
+ ("start", tuple(kwargs.get("extra_args") or ()), tuple(kwargs.get("offload") or ()))
+ )
+
+ def stop(self):
+ events.append(("stop",))
+
+ def is_alive(self):
+ return True
+
+ def img_gen(self, payload, **_kw):
+ if remaining["refusals"]:
+ remaining["refusals"] -= 1
+ raise RuntimeError(error)
+ return [_png_bytes()]
+
+ b = _loaded_backend()
+ b._state = _pinned_state(b, policy_flags = policy_flags, device = device)
+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server"))
+ object.__setattr__(b._state, "mode", "server")
+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)
+ monkeypatch.setattr(
+ bk,
+ "sd_cpp_graph_cut_options",
+ lambda binary: frozenset({"--max-vram"}) if max_vram_supported else frozenset(),
+ )
+ monkeypatch.setattr(bk, "_layer_split_supported", lambda binary: per_device_supported)
+ return b, events
+
+
+def test_a_vram_refusal_restarts_once_under_half_the_planned_free_vram(monkeypatch):
+ b, events = _vram_refusal_backend(monkeypatch, refusals = 1)
+ flags = b._state.offload_flags
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1
+ # Half of 6666 MiB, rounded down to 0.01 GiB.
+ assert [e for e in events if e[0] == "start"] == [("start", ("--max-vram", "3.25"), flags)]
+ assert b._state.vram_budget == "3.25"
+ assert b._state.device == "cuda" and b._state.offload_flags == flags
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1
+ assert len([e for e in events if e[0] == "start"]) == 1
+
+
+def test_a_vram_refusal_budgets_each_visible_card_from_its_own_free_vram(monkeypatch):
+ b, events = _vram_refusal_backend(
+ monkeypatch, refusals = 1, planned_free_mib = {"CUDA0": 45979, "CUDA1": 5771}
+ )
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1
+ assert [e[1] for e in events if e[0] == "start"] == [("--max-vram", "CUDA0=22.45,CUDA1=2.81")]
+
+
+def test_a_vram_refusal_leaves_a_full_bystander_card_to_sd_cpp(monkeypatch):
+ b, events = _vram_refusal_backend(
+ monkeypatch, refusals = 1, planned_free_mib = {"CUDA0": 1800, "CUDA1": 6666}
+ )
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1
+ assert [e[1] for e in events if e[0] == "start"] == [("--max-vram", "CUDA1=3.25")]
+
+
+def test_a_second_vram_refusal_in_the_same_request_is_surfaced(monkeypatch):
+ b, events = _vram_refusal_backend(monkeypatch, refusals = 2)
+ with pytest.raises(RuntimeError, match = "failed during weight preparation"):
+ b._generate_server(b._state, **_gen_kwargs())
+ assert len([e for e in events if e[0] == "start"]) == 1
+
+
+def test_a_load_recovers_once_another_application_frees_the_card(monkeypatch):
+ # The restart planned while the card was busy (466 MiB); the next request restarts under the first plan's budget.
+ b, events = _vram_refusal_backend(
+ monkeypatch,
+ refusals = 3,
+ restarted_plans = ({"CUDA1": 466}, {"CUDA1": 6666}),
+ )
+ with pytest.raises(RuntimeError, match = "failed during weight preparation"):
+ b._generate_server(b._state, **_gen_kwargs())
+ assert b._state.vram_budget == "3.25"
+ assert b._state.server.planned_free_vram_mib == {"CUDA1": 466}
+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())
+ assert len(images) == 1
+ assert [e[1] for e in events if e[0] == "start"] == [("--max-vram", "3.25")] * 2
+
+
+ "kwargs",
+ [
+ {"planned_free_mib": {}},
+ {"planned_free_mib": {"CUDA1": 1800}},
+ {"planned_free_mib": {"CUDA0": 45979, "CUDA1": 1800}},
+ {"planned_free_mib": {"CUDA0": 6666}},
+ {"max_vram_supported": False},
+ {"planned_free_mib": {"CUDA0": 45979, "CUDA1": 5771}, "per_device_supported": False},
+ {"device": "cpu"},
+ {"error": "sd-server generation failed: generation_failed: invalid sampler"},
+ {"error": _ALLOCATION_FAILURE.format(stage = "workspace allocation")},
+ {"error": _ALLOCATION_FAILURE.format(stage = "weight preparation")},
+ {
+ "error": _VRAM_REFUSAL.replace(
+ "Last output:\n",
+ "Last output:\n[E] ggml: ggml_backend_cpu_buffer_type_alloc_buffer: failed to allocate buffer of "
+ "size 220288512\n",
+ )
+ },
+ ],
+ ids = [
+ "no_plan",
+ "below_floor",
+ "render_card_below_floor",
+ "render_card_not_planned",
+ "no_max_vram",
+ "no_per_device_budgets",
+ "cpu",
+ "other_error",
+ "workspace_allocation_failure",
+ "weight_staging_allocation_failure",
+ "host_allocation_failure",
+ ],
+)
+def test_a_vram_refusal_without_a_usable_budget_is_surfaced_unchanged(monkeypatch, kwargs):
+ b, events = _vram_refusal_backend(monkeypatch, refusals = 1, **kwargs)
+ with pytest.raises(RuntimeError):
+ b._generate_server(b._state, **_gen_kwargs())
+ assert not [e for e in events if e[0] in ("start", "stop")]
+ assert b._state.vram_budget is None
+
+
+def test_a_device_loss_respawn_keeps_the_reduced_vram_budget(monkeypatch):
+ b, events = _vram_refusal_backend(monkeypatch, refusals = 0, device = "vulkan")
+ object.__setattr__(b._state, "vram_budget", "3.25")
+ server = b._respawn_server_after_device_loss(
+ b._state, "sd-server job failed: vk::Queue::submit: ErrorDeviceLost", threading.Event()
+ )
+ assert server is not None
+ assert [e[1] for e in events if e[0] == "start"] == [("--max-vram", "3.25")]
+
+
@pytest.mark.parametrize("why", ["explicit_gpu", "no_cpu_rung", "fallback_disabled"])
def test_a_repeated_vulkan_device_loss_without_a_cpu_rung_errors_with_a_hint(monkeypatch, why):
# A user-picked GPU is never moved to CPU behind their back, and an Arc / ROCm host (no CPU rung) or a disabled
diff --git a/studio/backend/tests/test_sd_cpp_vram_budget.py b/studio/backend/tests/test_sd_cpp_vram_budget.py
index 11b0db90636..5aca47bf762 100644
--- a/studio/backend/tests/test_sd_cpp_vram_budget.py
+++ b/studio/backend/tests/test_sd_cpp_vram_budget.py
@@ -951,3 +951,86 @@ def test_h3_card_probes_never_fall_back_to_a_torch_context(monkeypatch):
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)
assert video_mod._h3_card_free_bytes("cuda", 0, allow_context = False) is None
assert touched == []
+
+
+def test_image_vram_refusal_text_is_the_model_manager_declining_a_segment():
+ segment = (
+ "[E] MOSS-Image segment 1/43 (moss_image.layers.0) failed during {} --- ggml_runner.cpp:934"
+ )
+ for stage in ("weight preparation", "workspace capacity check", "allocated capacity check"):
+ assert args_mod.is_vram_refusal_text(segment.format(stage))
+ # The model manager's own warning beside it is not an allocation failure.
+ assert args_mod.is_vram_refusal_text(
+ "[W] model manager cannot make enough memory available on CUDA0: need 512.63 MB device / 0.63 MB "
+ "budget, available 0.00 MB device / 5549.38 MB budget --- model_manager.cpp:2101\n"
+ + segment.format("weight preparation")
+ )
+ assert not args_mod.is_vram_refusal_text("generation_failed: invalid sampler")
+ assert not args_mod.is_vram_refusal_text(None)
+
+
+def test_image_vram_refusal_text_leaves_failed_allocations_alone():
+ segment = "[E] MOSS-Image segment 12/43 (moss_image.layers.11) failed during {} --- ggml_runner.cpp:934"
+ allocator = (
+ "[E] ggml: ggml_backend_cuda_buffer_type_alloc_buffer: allocating 210.08 MiB on device 0: "
+ "cudaMalloc failed: out of memory\n"
+ "[E] ggml: alloc_tensor_range: failed to allocate CUDA0 buffer of size 220288512\n"
+ )
+ assert not args_mod.is_vram_refusal_text(segment.format("workspace allocation"))
+ assert not args_mod.is_vram_refusal_text(allocator + segment.format("workspace allocation"))
+ assert not args_mod.is_vram_refusal_text(allocator + segment.format("weight preparation"))
+ assert not args_mod.is_vram_refusal_text(
+ "vk::Device::allocateMemory: ErrorOutOfDeviceMemory\n"
+ + segment.format("weight preparation")
+ )
+ assert not args_mod.is_vram_refusal_text(_AUDIT_OOM)
+ for host in (
+ "[E] ggml: ggml_backend_cpu_buffer_type_alloc_buffer: failed to allocate buffer of size 220288512\n",
+ "[E] model manager alloc params backend buffer failed, size = 210.08MB --- model_manager.cpp:964\n",
+ "[E] vk::Device::allocateMemory: ErrorOutOfHostMemory\n",
+ "[E] ggml_vulkan: host malloc failed\n",
+ ):
+ assert not args_mod.is_vram_refusal_text(host + segment.format("weight preparation"))
+ assert args_mod.is_device_oom_text(_AUDIT_OOM)
+ assert args_mod.is_device_oom_text(segment.format("workspace allocation"))
+ assert not args_mod.is_device_oom_text(segment.format("weight preparation"))
+
+
+def test_image_vram_retry_budget_is_half_the_planned_free_vram():
+ assert args_mod.image_vram_retry_budget({"CUDA0": 6666}) == "3.25"
+ assert args_mod.image_vram_retry_budget({"Vulkan0": 24176}) == "11.80"
+ assert args_mod.image_vram_retry_budget({"CUDA0": 2048}) == "1.00"
+ assert args_mod.image_vram_retry_budget({"CUDA0": 2047}) is None
+ assert args_mod.image_vram_retry_budget({"CUDA0": 0}) is None
+ assert args_mod.image_vram_retry_budget({}) is None
+ assert args_mod.image_vram_retry_budget(None) is None
+
+
+def test_image_vram_retry_budget_is_per_card_and_needs_the_render_cards():
+ plan = {"CUDA0": 45979, "CUDA1": 5771}
+ assert args_mod.image_vram_retry_budget(plan) == "CUDA0=22.45,CUDA1=2.81"
+ assert args_mod.image_vram_retry_budget(plan, {"cuda1"}) == "CUDA0=22.45,CUDA1=2.81"
+ # A card under the floor is left out, unless the render is pinned to it or it is missing from the plan.
+ assert (
+ args_mod.image_vram_retry_budget({"CUDA0": 45979, "CUDA1": 1800}, {"cuda0"})
+ == "CUDA0=22.45"
+ )
+ assert args_mod.image_vram_retry_budget({"CUDA0": 45979, "CUDA1": 1800}, {"cuda1"}) is None
+ assert args_mod.image_vram_retry_budget({"CUDA0": 45979}, {"cuda1"}) is None
+
+
+def test_planned_free_vram_is_read_from_the_auto_fit_device_lines():
+ plan = (
+ "[I] auto-fit plan: --- backend_fit.cpp:334",
+ "[I] devices: --- backend_fit.cpp:335",
+ "[I] CUDA0 NVIDIA RTX 6000 Ada Generation free 45979 MiB, budget 45467 MiB --- backend_fit.cpp:337",
+ "[I] CUDA1 NVIDIA GeForce RTX 3090 free 5771 MiB, budget 5259 MiB --- backend_fit.cpp:337",
+ "[I] RAM free 69220 MiB, params budget 62298 MiB --- backend_fit.cpp:344",
+ "[I] DiT params 5228 MiB, compute reserve 2048 MiB -> compute CUDA0, params CUDA0 --- backend_fit.cpp:357",
+ )
+ found = {}
+ for line in plan:
+ match = srv._AUTO_FIT_DEVICE_FREE_RE.search(line)
+ if match is not None:
+ found[match.group(1)] = int(match.group(2))
+ assert found == {"CUDA0": 45979, "CUDA1": 5771}