Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio sd.cpp: retry an image render that sd-server refused for VRAM under half the budget (#17339)
oobaboogaf2e923eb95a2
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.pyindex e0c9693d811..2df626dcdf5 100644--- a/studio/backend/core/inference/sd_cpp_args.py+++ b/studio/backend/core/inference/sd_cpp_args.py@@ -8,7 +8,7 @@ from __future__ import annotationsimport functoolsimport os-from collections.abc import Sequence+from collections.abc import Iterable, Mapping, Sequencefrom dataclasses import dataclass, fieldfrom typing import Optional@@ -709,6 +709,60 @@ def is_device_oom_text(text: Optional[str]) -> bool:return False+# sd.cpp ``fail_segment`` capacity-check stages (ggml_runner.cpp); "workspace allocation" is an allocator failure.+_SEGMENT_REFUSAL_MARKERS: tuple[str, ...] = (+ "failed during weight preparation",+ "failed during workspace capacity check",+ "failed during allocated capacity check",+)+# Allocator failures on the host RAM that weight preparation stages into, as well as on the device.+_ALLOCATION_FAILURE_MARKERS: tuple[str, ...] = (+ *_HOST_OOM_MARKERS,+ "erroroutofhostmemory",+ "failed to allocate",+ "backend buffer failed",+)+# sd.cpp fills free-512 MiB with cached weights, so each phase needs memory the last one released; half leaves it+# unclaimed when the driver reports the release late (Windows may defer destroying allocations).+IMAGE_VRAM_RETRY_SCALE = 0.5+++def is_vram_refusal_text(text: Optional[str]) -> bool:+ """sd.cpp's model manager declined a segment for VRAM, and no allocation failed outright (that keeps its error)."""+ lines = str(text or "").lower().splitlines()+ if not any(marker in line for line in lines for marker in _SEGMENT_REFUSAL_MARKERS):+ return False+ return not any(+ "failed during" not in line+ and (+ is_device_oom_text(line)+ or any(marker in line for marker in _ALLOCATION_FAILURE_MARKERS)+ )+ for line in lines+ )+++def image_vram_retry_budget(+ planned_free_mib: Optional[Mapping[str, int]], render_devices: Iterable[str] = ()+) -> Optional[str]:+ """``--max-vram`` for the image retry: half of each planned card's free VRAM (``CUDA0=..,CUDA1=..`` for several),+ skipping cards under the floor. None if none qualifies or a ``render_devices`` card does not."""+ planned = dict(planned_free_mib or {})+ budgets: dict[str, float] = {}+ for name, mib in planned.items():+ gib = graph_cut_budget_gib(int(mib) << 20, reserve_bytes = 0, scale = IMAGE_VRAM_RETRY_SCALE)+ if gib is not None:+ budgets[name] = gib+ if not budgets:+ return None+ known = {name.lower() for name in budgets}+ if any(str(device).lower() not in known for device in render_devices):+ return None+ if len(planned) == 1:+ return _format_gib(next(iter(budgets.values())))+ return ",".join(f"{name}={_format_gib(gib)}" for name, gib in budgets.items())++def device_backend_flags(device_name: Optional[str],offload: Optional[list[str]] = None,diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex d5434f7ac95..f1477ec8a80 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -2930,10 +2930,14 @@ class _SdState:gpu_explicit: bool = Falsemulti_gpu: Optional[dict[str, Any]] = Nonesplit_physical_gpu_id: Optional[int] = None+ vram_budget: Optional[str] = Nonedef spawn_env(self) -> Optional[dict[str, str]]:return dict(self.child_env) or None+ def vram_budget_args(self) -> list[str]:+ return ["--max-vram", self.vram_budget] if self.vram_budget else []+@propertydef placed_ordinal(self) -> Optional[int]:"""Torch ordinal of ``physical_gpu_id`` (a split's diffusion card); without it card-sharing helpers (upsampler) split over all GPUs."""@@ -5027,6 +5031,7 @@ class SdCppDiffusionBackend:# One same-build respawn per request; a second loss moves this load to CPU where the rung applies. Persists nothing.device_respawned = Falsedevice_losses = 0+ vram_restarted = Falsehires_payload: Optional[dict] = Nonehires_staged: list[Path] = []@@ -5099,10 +5104,12 @@ class SdCppDiffusionBackend:except RuntimeError as first_exc:# A ggml unsupported-op abort killed the server: this graph cannot run on the GPU backend at all,# so restart the model on the CPU backend once and retry this chunk. A lost Vulkan device needs a- # fresh device: respawn the same build once. Any other death propagates.+ # fresh device: respawn the same build once. A VRAM refusal restarts once under a smaller+ # budget. Any other death propagates.exc = first_excwhile True:moved_to_cpu = False+ vram_budget: Optional[str] = Noneserver = self._restart_server_on_cpu_backend(state, str(exc), cancel)if server is None and output_shows_vulkan_device_lost(str(exc)):device_losses += 1@@ -5120,10 +5127,17 @@ class SdCppDiffusionBackend:state, str(exc), cancel)device_respawned = server is not None+ if server is None and not vram_restarted:+ server, vram_budget = self._restart_server_with_vram_headroom(+ state, str(exc), cancel+ )+ vram_restarted = server is not Noneif server is None:raise excold_server = state.serverstate = replace(state, server = server)+ if vram_budget is not None:+ state = replace(state, vram_budget = vram_budget)if moved_to_cpu:# Placement follows the server: the flags the CPU server got, no Vulkan pin.state = replace(@@ -5287,7 +5301,59 @@ class SdCppDiffusionBackend:native_speed = state.native_speed,threads = state.threads,env = state.spawn_env(),+ extra_args = state.vram_budget_args(),+ )++ def _restart_server_with_vram_headroom(+ self, state: _SdState, error_text: str, cancel: threading.Event+ ) -> tuple[Optional[SdCppServer], Optional[str]]:+ """Restart sd-server under ``--max-vram`` after sd.cpp refused a render for VRAM; returns the server and its+ budget, or ``(None, None)``. The budget comes from the first refusing server's plan and is reused: a server+ restarted while another application held the card planned too little to halve, and keeps refusing."""+ if not sd_cpp_args_mod.is_vram_refusal_text(error_text):+ return None, None+ if state.device == "cpu" or self._cpu_backend_forced:+ return None, None+ if state.server is None or cancel.is_set():+ return None, None+ binary = state.server.binary+ # A two-card split already carries per-device budgets, and an older build rejects the flag.+ if "--max-vram" in state.offload_flags or "--max-vram" not in sd_cpp_graph_cut_options(+ binary+ ):+ return None, None+ planned = dict(getattr(state.server, "planned_free_vram_mib", None) or {})+ budget = state.vram_budget or sd_cpp_args_mod.image_vram_retry_budget(+ planned, _backend_devices(state.offload_flags)+ )+ if budget is None or ("=" in budget and not _layer_split_supported(binary)):+ return None, None+ if not self._begin_replacement(state):+ return None, None+ logger.warning(+ "sd-server refused the render for VRAM; restarting it with --max-vram %s (half the free VRAM this "+ "load first planned with; the refusing server planned with, in MiB: %s) and retrying once. "+ "Details: %s",+ budget,+ planned,+ error_text[:300],+ )+ state.server.stop()+ server = SdCppServer(binary)+ with self._lock:+ self._pending_server = server+ started = self._start_replacement(+ state,+ server,+ cancel,+ vae_format = state.vae_format,+ offload = state.offload_flags,+ native_speed = state.native_speed,+ threads = state.threads,+ env = state.spawn_env(),+ extra_args = ["--max-vram", budget],)+ return started, (budget if started is not None else None)def _restart_server_on_cpu_backend(self,diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.pyindex a8d3fdc7739..abfac256399 100644--- a/studio/backend/core/inference/sd_cpp_server.py+++ b/studio/backend/core/inference/sd_cpp_server.py@@ -65,6 +65,11 @@ _TOTAL_PARAMS_VRAM_RE = re.compile(re.IGNORECASE,)+# sd.cpp auto-fit device line "[I] CUDA0 <card> free N MiB, budget M MiB"; the RAM "params budget" line won't match.+_AUTO_FIT_DEVICE_FREE_RE = re.compile(+ r"^\[I\]\s+(\S+)\s.*\bfree\s+([0-9]+)\s*MiB,\s*budget\s+[0-9]+\s*MiB\b"+)+_TERMINAL_OK = "completed"_TERMINAL_FAIL = "failed"_TERMINAL_CANCELLED = "cancelled"@@ -165,6 +170,7 @@ class SdCppServer:self.port: Optional[int] = Noneself._process: Optional[subprocess.Popen] = Noneself._resident_params_vram_gb: Optional[float] = None+ self._planned_free_vram_mib: dict[str, int] = {}# Bounded tail buffer shared by the drain thread (appends) and readers (diagnostics).self._tail: deque[str] = deque(maxlen = 200)self._lines_seen = 0@@ -267,6 +273,7 @@ class SdCppServer:# Clear in place; reassigning [] would drop the maxlen bound and grow unbounded.self._tail.clear()self._resident_params_vram_gb = None+ self._planned_free_vram_mib = {}self._spawn_error: Optional[Exception] = Nonespawned = threading.Event()@@ -468,6 +475,9 @@ class SdCppServer:value = float(match.group(1)) / 1024.0if value > 0:self._resident_params_vram_gb = value+ match = _AUTO_FIT_DEVICE_FREE_RE.search(line)+ if match is not None:+ self._planned_free_vram_mib[match.group(1)] = int(match.group(2))logger.debug("[sd-server] %s", line)cb = self._step_listenerif cb is not None:@@ -486,6 +496,11 @@ class SdCppServer:if wake is not None:wake.set()+ @property+ def planned_free_vram_mib(self) -> dict[str, int]:+ """Free VRAM per device (``{"CUDA0": 6666}``) from sd.cpp's startup auto-fit plan; empty without one."""+ return dict(self._planned_free_vram_mib)+def reclaimable_params_vram_gb(self, physical_gpu_id: Optional[int]) -> Optional[dict[int, float]]:diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.pyindex 249918cc39d..d39b8ac33bd 100644--- a/studio/backend/tests/test_sd_cpp_backend.py+++ b/studio/backend/tests/test_sd_cpp_backend.py@@ -3133,6 +3133,192 @@ def test_a_single_vulkan_device_loss_stays_on_vulkan(monkeypatch):assert not persisted+_VRAM_REFUSAL = (+ "sd-server generation failed: generation_failed: generate_image returned no results\n"+ "Last output:\n"+ "[E] MOSS-Image segment 1/43 (moss_image.layers.0) failed during weight preparation --- ggml_runner.cpp:934\n"+ "[E] diffusion model compute failed --- diffusion_engine.cpp:2729\n"+ "[E] Diffusion model sampling failed --- diffusion_engine.cpp:2976\n"+ "[E] sampling for image 1/1 failed after 0.09s --- image.cpp:1007"+)+++_ALLOCATION_FAILURE = (+ "sd-server generation failed: generation_failed: generate_image returned no results\n"+ "Last output:\n"+ "[E] ggml: ggml_backend_cuda_buffer_type_alloc_buffer: allocating 210.08 MiB on device 0: "+ "cudaMalloc failed: out of memory\n"+ "[E] ggml: alloc_tensor_range: failed to allocate CUDA0 buffer of size 220288512\n"+ "[E] MOSS-Image segment 12/43 (moss_image.layers.11) failed during {stage} --- ggml_runner.cpp:934\n"+ "[E] diffusion model compute failed --- diffusion_engine.cpp:2729"+)+++def _vram_refusal_backend(+ monkeypatch,+ *,+ refusals,+ planned_free_mib = None,+ restarted_plans = (),+ error = _VRAM_REFUSAL,+ max_vram_supported = True,+ per_device_supported = True,+ policy_flags = (),+ device = "cuda",+):+ """CUDA load pinned to CUDA1 whose first ``refusals`` jobs fail with ``error``; ``restarted_plans`` are the+ free-VRAM plans each replacement server reads in turn."""+ events: list = []+ remaining = {"refusals": refusals}+ planned = {"CUDA1": 6666} if planned_free_mib is None else planned_free_mib+ plans = [planned, *restarted_plans]++ class _FakeServer:+ lora_dir = None++ def __init__(self, binary):+ self.binary = binary+ self.planned_free_vram_mib = dict(plans.pop(0) if len(plans) > 1 else plans[0])++ def start(self, files, **kwargs):+ events.append(+ ("start", tuple(kwargs.get("extra_args") or ()), tuple(kwargs.get("offload") or ()))+ )++ def stop(self):+ events.append(("stop",))++ def is_alive(self):+ return True++ def img_gen(self, payload, **_kw):+ if remaining["refusals"]:+ remaining["refusals"] -= 1+ raise RuntimeError(error)+ return [_png_bytes()]++ b = _loaded_backend()+ b._state = _pinned_state(b, policy_flags = policy_flags, device = device)+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server"))+ object.__setattr__(b._state, "mode", "server")+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)+ monkeypatch.setattr(+ bk,+ "sd_cpp_graph_cut_options",+ lambda binary: frozenset({"--max-vram"}) if max_vram_supported else frozenset(),+ )+ monkeypatch.setattr(bk, "_layer_split_supported", lambda binary: per_device_supported)+ return b, events+++def test_a_vram_refusal_restarts_once_under_half_the_planned_free_vram(monkeypatch):+ b, events = _vram_refusal_backend(monkeypatch, refusals = 1)+ flags = b._state.offload_flags+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1+ # Half of 6666 MiB, rounded down to 0.01 GiB.+ assert [e for e in events if e[0] == "start"] == [("start", ("--max-vram", "3.25"), flags)]+ assert b._state.vram_budget == "3.25"+ assert b._state.device == "cuda" and b._state.offload_flags == flags+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1+ assert len([e for e in events if e[0] == "start"]) == 1+++def test_a_vram_refusal_budgets_each_visible_card_from_its_own_free_vram(monkeypatch):+ b, events = _vram_refusal_backend(+ monkeypatch, refusals = 1, planned_free_mib = {"CUDA0": 45979, "CUDA1": 5771}+ )+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1+ assert [e[1] for e in events if e[0] == "start"] == [("--max-vram", "CUDA0=22.45,CUDA1=2.81")]+++def test_a_vram_refusal_leaves_a_full_bystander_card_to_sd_cpp(monkeypatch):+ b, events = _vram_refusal_backend(+ monkeypatch, refusals = 1, planned_free_mib = {"CUDA0": 1800, "CUDA1": 6666}+ )+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1+ assert [e[1] for e in events if e[0] == "start"] == [("--max-vram", "CUDA1=3.25")]+++def test_a_second_vram_refusal_in_the_same_request_is_surfaced(monkeypatch):+ b, events = _vram_refusal_backend(monkeypatch, refusals = 2)+ with pytest.raises(RuntimeError, match = "failed during weight preparation"):+ b._generate_server(b._state, **_gen_kwargs())+ assert len([e for e in events if e[0] == "start"]) == 1+++def test_a_load_recovers_once_another_application_frees_the_card(monkeypatch):+ # The restart planned while the card was busy (466 MiB); the next request restarts under the first plan's budget.+ b, events = _vram_refusal_backend(+ monkeypatch,+ refusals = 3,+ restarted_plans = ({"CUDA1": 466}, {"CUDA1": 6666}),+ )+ with pytest.raises(RuntimeError, match = "failed during weight preparation"):+ b._generate_server(b._state, **_gen_kwargs())+ assert b._state.vram_budget == "3.25"+ assert b._state.server.planned_free_vram_mib == {"CUDA1": 466}+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1+ assert [e[1] for e in events if e[0] == "start"] == [("--max-vram", "3.25")] * 2+++ "kwargs",+ [+ {"planned_free_mib": {}},+ {"planned_free_mib": {"CUDA1": 1800}},+ {"planned_free_mib": {"CUDA0": 45979, "CUDA1": 1800}},+ {"planned_free_mib": {"CUDA0": 6666}},+ {"max_vram_supported": False},+ {"planned_free_mib": {"CUDA0": 45979, "CUDA1": 5771}, "per_device_supported": False},+ {"device": "cpu"},+ {"error": "sd-server generation failed: generation_failed: invalid sampler"},+ {"error": _ALLOCATION_FAILURE.format(stage = "workspace allocation")},+ {"error": _ALLOCATION_FAILURE.format(stage = "weight preparation")},+ {+ "error": _VRAM_REFUSAL.replace(+ "Last output:\n",+ "Last output:\n[E] ggml: ggml_backend_cpu_buffer_type_alloc_buffer: failed to allocate buffer of "+ "size 220288512\n",+ )+ },+ ],+ ids = [+ "no_plan",+ "below_floor",+ "render_card_below_floor",+ "render_card_not_planned",+ "no_max_vram",+ "no_per_device_budgets",+ "cpu",+ "other_error",+ "workspace_allocation_failure",+ "weight_staging_allocation_failure",+ "host_allocation_failure",+ ],+)+def test_a_vram_refusal_without_a_usable_budget_is_surfaced_unchanged(monkeypatch, kwargs):+ b, events = _vram_refusal_backend(monkeypatch, refusals = 1, **kwargs)+ with pytest.raises(RuntimeError):+ b._generate_server(b._state, **_gen_kwargs())+ assert not [e for e in events if e[0] in ("start", "stop")]+ assert b._state.vram_budget is None+++def test_a_device_loss_respawn_keeps_the_reduced_vram_budget(monkeypatch):+ b, events = _vram_refusal_backend(monkeypatch, refusals = 0, device = "vulkan")+ object.__setattr__(b._state, "vram_budget", "3.25")+ server = b._respawn_server_after_device_loss(+ b._state, "sd-server job failed: vk::Queue::submit: ErrorDeviceLost", threading.Event()+ )+ assert server is not None+ assert [e[1] for e in events if e[0] == "start"] == [("--max-vram", "3.25")]++@pytest.mark.parametrize("why", ["explicit_gpu", "no_cpu_rung", "fallback_disabled"])def test_a_repeated_vulkan_device_loss_without_a_cpu_rung_errors_with_a_hint(monkeypatch, why):# A user-picked GPU is never moved to CPU behind their back, and an Arc / ROCm host (no CPU rung) or a disableddiff --git a/studio/backend/tests/test_sd_cpp_vram_budget.py b/studio/backend/tests/test_sd_cpp_vram_budget.pyindex 11b0db90636..5aca47bf762 100644--- a/studio/backend/tests/test_sd_cpp_vram_budget.py+++ b/studio/backend/tests/test_sd_cpp_vram_budget.py@@ -951,3 +951,86 @@ def test_h3_card_probes_never_fall_back_to_a_torch_context(monkeypatch):assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)assert video_mod._h3_card_free_bytes("cuda", 0, allow_context = False) is Noneassert touched == []+++def test_image_vram_refusal_text_is_the_model_manager_declining_a_segment():+ segment = (+ "[E] MOSS-Image segment 1/43 (moss_image.layers.0) failed during {} --- ggml_runner.cpp:934"+ )+ for stage in ("weight preparation", "workspace capacity check", "allocated capacity check"):+ assert args_mod.is_vram_refusal_text(segment.format(stage))+ # The model manager's own warning beside it is not an allocation failure.+ assert args_mod.is_vram_refusal_text(+ "[W] model manager cannot make enough memory available on CUDA0: need 512.63 MB device / 0.63 MB "+ "budget, available 0.00 MB device / 5549.38 MB budget --- model_manager.cpp:2101\n"+ + segment.format("weight preparation")+ )+ assert not args_mod.is_vram_refusal_text("generation_failed: invalid sampler")+ assert not args_mod.is_vram_refusal_text(None)+++def test_image_vram_refusal_text_leaves_failed_allocations_alone():+ segment = "[E] MOSS-Image segment 12/43 (moss_image.layers.11) failed during {} --- ggml_runner.cpp:934"+ allocator = (+ "[E] ggml: ggml_backend_cuda_buffer_type_alloc_buffer: allocating 210.08 MiB on device 0: "+ "cudaMalloc failed: out of memory\n"+ "[E] ggml: alloc_tensor_range: failed to allocate CUDA0 buffer of size 220288512\n"+ )+ assert not args_mod.is_vram_refusal_text(segment.format("workspace allocation"))+ assert not args_mod.is_vram_refusal_text(allocator + segment.format("workspace allocation"))+ assert not args_mod.is_vram_refusal_text(allocator + segment.format("weight preparation"))+ assert not args_mod.is_vram_refusal_text(+ "vk::Device::allocateMemory: ErrorOutOfDeviceMemory\n"+ + segment.format("weight preparation")+ )+ assert not args_mod.is_vram_refusal_text(_AUDIT_OOM)+ for host in (+ "[E] ggml: ggml_backend_cpu_buffer_type_alloc_buffer: failed to allocate buffer of size 220288512\n",+ "[E] model manager alloc params backend buffer failed, size = 210.08MB --- model_manager.cpp:964\n",+ "[E] vk::Device::allocateMemory: ErrorOutOfHostMemory\n",+ "[E] ggml_vulkan: host malloc failed\n",+ ):+ assert not args_mod.is_vram_refusal_text(host + segment.format("weight preparation"))+ assert args_mod.is_device_oom_text(_AUDIT_OOM)+ assert args_mod.is_device_oom_text(segment.format("workspace allocation"))+ assert not args_mod.is_device_oom_text(segment.format("weight preparation"))+++def test_image_vram_retry_budget_is_half_the_planned_free_vram():+ assert args_mod.image_vram_retry_budget({"CUDA0": 6666}) == "3.25"+ assert args_mod.image_vram_retry_budget({"Vulkan0": 24176}) == "11.80"+ assert args_mod.image_vram_retry_budget({"CUDA0": 2048}) == "1.00"+ assert args_mod.image_vram_retry_budget({"CUDA0": 2047}) is None+ assert args_mod.image_vram_retry_budget({"CUDA0": 0}) is None+ assert args_mod.image_vram_retry_budget({}) is None+ assert args_mod.image_vram_retry_budget(None) is None+++def test_image_vram_retry_budget_is_per_card_and_needs_the_render_cards():+ plan = {"CUDA0": 45979, "CUDA1": 5771}+ assert args_mod.image_vram_retry_budget(plan) == "CUDA0=22.45,CUDA1=2.81"+ assert args_mod.image_vram_retry_budget(plan, {"cuda1"}) == "CUDA0=22.45,CUDA1=2.81"+ # A card under the floor is left out, unless the render is pinned to it or it is missing from the plan.+ assert (+ args_mod.image_vram_retry_budget({"CUDA0": 45979, "CUDA1": 1800}, {"cuda0"})+ == "CUDA0=22.45"+ )+ assert args_mod.image_vram_retry_budget({"CUDA0": 45979, "CUDA1": 1800}, {"cuda1"}) is None+ assert args_mod.image_vram_retry_budget({"CUDA0": 45979}, {"cuda1"}) is None+++def test_planned_free_vram_is_read_from_the_auto_fit_device_lines():+ plan = (+ "[I] auto-fit plan: --- backend_fit.cpp:334",+ "[I] devices: --- backend_fit.cpp:335",+ "[I] CUDA0 NVIDIA RTX 6000 Ada Generation free 45979 MiB, budget 45467 MiB --- backend_fit.cpp:337",+ "[I] CUDA1 NVIDIA GeForce RTX 3090 free 5771 MiB, budget 5259 MiB --- backend_fit.cpp:337",+ "[I] RAM free 69220 MiB, params budget 62298 MiB --- backend_fit.cpp:344",+ "[I] DiT params 5228 MiB, compute reserve 2048 MiB -> compute CUDA0, params CUDA0 --- backend_fit.cpp:357",+ )+ found = {}+ for line in plan:+ match = srv._AUTO_FIT_DEVICE_FREE_RE.search(line)+ if match is not None:+ found[match.group(1)] = int(match.group(2))+ assert found == {"CUDA0": 45979, "CUDA1": 5771}