Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""#8235: an AUTOMATIC image / video load ranks the visible GPUs by free VRAM.
The picker on "auto" sends no gpu_ids, and bare "cuda" is the current device, PCI
slot 0 under Studio's PCI_BUS_ID order, so an 8 GB card 0 beside an idle 24 GB
card 1 budgeted the load against 8 GB and offloaded to RAM. Explicit picks and
masks keep winning; one visible card, UUID masks and ROCm are unchanged."""
import ast
import asyncio
import types
from pathlib import Path
import pytest
from core.inference import diffusion_device as dd
from utils.hardware import hardware as hw
def _dev(index, total, used):
return {"index": index, "vram_total_gb": total, "vram_used_gb": used}
def _host(
monkeypatch,
visible,
devices,
rocm = False,
order_matches = True,
caps = None,
):
from utils.hardware import nvidia
if caps is None:
caps = {d["index"]: (8, 6) for d in devices}
monkeypatch.setattr(nvidia, "get_compute_capabilities", lambda: dict(caps))
monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
monkeypatch.setattr(hw, "IS_ROCM", rocm)
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: list(visible))
monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: order_matches)
monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda ids: ({}, "index"))
monkeypatch.setattr(hw, "_get_parent_visible_gpu_spec", lambda: {"numeric_ids": list(visible)})
smi = {"available": True, "devices": devices}
monkeypatch.setattr(hw, "_smi_visible_utilization", lambda spec: smi)
monkeypatch.setattr(hw, "get_visible_gpu_utilization", lambda: smi)
def test_auto_picks_the_card_with_the_most_free_vram(monkeypatch):
_host(monkeypatch, [0, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)])
assert dd.resolve_auto_cuda_ordinal() == 1
def test_auto_maps_physical_ids_through_the_mask(monkeypatch):
# CUDA_VISIBLE_DEVICES=5,2: physical 2 is torch ordinal 1.
_host(monkeypatch, [5, 2], [_dev(2, 80.0, 0.0), _dev(5, 80.0, 70.0)])
assert dd.resolve_auto_cuda_ordinal() == 1
def test_auto_ties_keep_the_current_device(monkeypatch):
# An idle uniform host stays on card 0, MiB-level noise included, and as None (no explicit pin).
_host(monkeypatch, [0, 1, 2], [_dev(0, 80.0, 0.6), _dev(1, 80.0, 0.5), _dev(2, 80.0, 0.6)])
assert dd.resolve_auto_cuda_ordinal() is None
def test_auto_ties_among_other_cards_keep_the_lowest_ordinal(monkeypatch):
_host(monkeypatch, [0, 1, 2], [_dev(0, 80.0, 70.0), _dev(1, 80.0, 0.5), _dev(2, 80.0, 0.6)])
assert dd.resolve_auto_cuda_ordinal() == 1
@pytest.mark.parametrize(
"visible, devices, rocm",
[
([0], [_dev(0, 24.0, 1.0)], False), # one card: unchanged
([], [_dev(0, 24.0, 1.0), _dev(1, 24.0, 0.0)], False), # UUID/MIG mask
([0, 1], [_dev(0, 8.0, 7.0), _dev(1, 96.0, 0.0)], True), # ROCm APU pool
([0, 1], [], False), # no telemetry
],
)
def test_auto_keeps_the_current_device(monkeypatch, visible, devices, rocm):
_host(monkeypatch, visible, devices, rocm = rocm)
assert dd.resolve_auto_cuda_ordinal() is None
def test_auto_never_raises(monkeypatch):
_host(monkeypatch, [0, 1], [])
def _boom(*_a):
raise RuntimeError("nvidia-smi hung")
monkeypatch.setattr(hw, "_smi_visible_utilization", _boom)
assert dd.resolve_auto_cuda_ordinal() is None
@pytest.mark.parametrize("identical, expected", [(True, None), (False, 0), (None, 0)])
def test_auto_does_not_rank_when_cuda_order_is_not_smi_order(monkeypatch, identical, expected):
# CUDA_DEVICE_ORDER=FASTEST_FIRST: nvidia-smi rows are not torch ordinals, so no ranking. Identical
# cards keep None; otherwise card 0 stays explicit so a pooled thread cannot preflight elsewhere.
_host(monkeypatch, [0, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)], order_matches = False)
monkeypatch.setattr(hw, "_cuda_cards_identical", lambda gpu_ids = None: identical)
assert dd.resolve_auto_cuda_ordinal() is expected
def test_auto_never_falls_back_to_torch_telemetry(monkeypatch):
# No nvidia-smi: the hardware layer's torch fallback opens a CUDA context per card.
_host(monkeypatch, [0, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)])
monkeypatch.setattr(hw, "_smi_visible_utilization", lambda spec: None)
assert dd.resolve_auto_cuda_ordinal() is None
def _route(monkeypatch, device = "cuda"):
from routes import inference as routes
monkeypatch.setattr(
dd, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = device)
)
monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", lambda: 1)
monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: True)
# The event-loop gate reads live detection state; defer to the stubbed worker gate.
monkeypatch.setattr(dd, "auto_cuda_ranking_possible_cached", lambda: None)
monkeypatch.setattr(dd, "resolve_selected_cuda_ordinal", lambda ids, **_k: min(ids))
return routes._selected_gpu_ordinal
def test_route_ranks_an_automatic_load(monkeypatch):
resolve = _route(monkeypatch)
assert asyncio.run(resolve(None)) == 1
assert asyncio.run(resolve([])) == 1
def test_route_explicit_pick_still_wins(monkeypatch):
resolve = _route(monkeypatch)
assert asyncio.run(resolve([0])) == 0
def test_route_auto_stays_off_cuda_and_while_ranking_is_barred(monkeypatch):
resolve = _route(monkeypatch, device = "mps")
assert asyncio.run(resolve(None)) is None
resolve = _route(monkeypatch)
assert asyncio.run(resolve(None, allow_ranking = False)) is None
def test_auto_keeps_the_current_device_beside_an_integrated_gpu(monkeypatch):
# DGX Spark-class row: nvidia-smi says N/A (or a carve-out) for a part whose budget is host RAM.
_host(monkeypatch, [0, 1], [_dev(0, None, None), _dev(1, 24.0, 0.5)])
monkeypatch.setattr(
hw, "_integrated_cuda_rows", lambda ids: ({0: {"total_gb": 121.0}}, "index")
)
assert dd.resolve_auto_cuda_ordinal() is None
def test_backend_fallback_honours_a_pinned_none(monkeypatch):
# The auto-switch planned "current device"; begin_load's own fallback must not re-rank.
from core.inference import media_locality
_host(monkeypatch, [0, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)])
token = media_locality.pin_planned_gpu_ordinal(None)
try:
assert dd.resolve_auto_cuda_ordinal() is None
finally:
media_locality._PLANNED_ORDINAL.reset(token)
assert dd.resolve_auto_cuda_ordinal() == 1
def test_route_uses_the_ordinal_an_auto_switch_planned(monkeypatch):
from core.inference import media_locality
resolve = _route(monkeypatch)
token = media_locality.pin_planned_gpu_ordinal(0)
try:
assert media_locality.plan_gpu_ordinal() == 0
assert asyncio.run(resolve(None)) == 0
assert asyncio.run(resolve([1])) == 1 # an explicit pick still wins
finally:
media_locality._PLANNED_ORDINAL.reset(token)
assert asyncio.run(resolve(None)) == 1
def test_plan_sizes_the_card_the_load_will_rank(monkeypatch):
from core.inference import media_locality
monkeypatch.setattr(
dd, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = "cuda")
)
_host(monkeypatch, [0, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)])
assert media_locality.plan_gpu_ordinal() == 1
@pytest.mark.parametrize(
"rel",
["core/inference/diffusion.py", "core/inference/video.py", "core/inference/sd_cpp_backend.py"],
)
def test_direct_backend_loads_rank_automatic_too(rel):
# A direct caller (MCP, plugin) passes neither gpu_ids nor an ordinal.
src = (Path(dd.__file__).resolve().parents[2] / rel).read_text(encoding = "utf-8")
calls = {
node.func.id
for node in ast.walk(ast.parse(src))
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
}
assert "auto_cuda_ordinal_for_load" in calls
def test_auto_skips_a_freer_card_older_than_the_current_one(monkeypatch):
# The current sm_86 card works today; a freer Pascal may lack the wheel's kernels or bf16.
_host(
monkeypatch, [0, 1], [_dev(0, 24.0, 20.0), _dev(1, 8.0, 0.0)], caps = {0: (8, 6), 1: (6, 1)}
)
# Mixed cards keep card 0 explicit.
assert dd.resolve_auto_cuda_ordinal() == 0
def test_auto_a_newer_card_still_competes(monkeypatch):
_host(
monkeypatch, [0, 1], [_dev(0, 24.0, 20.0), _dev(1, 24.0, 0.0)], caps = {0: (8, 6), 1: (8, 9)}
)
assert dd.resolve_auto_cuda_ordinal() == 1
def test_auto_keeps_the_current_device_without_capabilities(monkeypatch):
_host(monkeypatch, [0, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)], caps = {})
assert dd.resolve_auto_cuda_ordinal() is None
def test_auto_keeps_the_current_device_when_a_mask_entry_names_no_card(monkeypatch):
# CUDA_VISIBLE_DEVICES=0,99,1 is just card 0 to CUDA, so physical 1 is no ordinal at all.
_host(monkeypatch, [0, 99, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)])
assert dd.resolve_auto_cuda_ordinal() is None
def test_route_pins_its_automatic_decision_for_the_backend(monkeypatch):
# begin_load's own fallback must reuse the route's answer (None included), not re-rank.
from routes import inference as routes
monkeypatch.setattr(
dd, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = "cuda")
)
answers = iter([None, 1])
monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", lambda: next(answers))
monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: True)
monkeypatch.setattr(dd, "auto_cuda_ranking_possible_cached", lambda: None)
real = dd.__dict__["_PLANNED_ORDINAL"]
async def load():
ordinal = await routes._selected_gpu_ordinal(None)
return ordinal, real.get()
assert asyncio.run(load()) == (None, None)
def test_automatic_primary_pick_ranks_by_the_budgeted_room(monkeypatch):
from utils import vram_budget_settings
fraction = {"value": vram_budget_settings.VRAM_FRACTION_DEFAULT}
monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: fraction["value"])
_host(monkeypatch, [0, 1], [_dev(0, 24.0, 1.0), _dev(1, 192.0, 132.0)])
assert dd.resolve_auto_cuda_ordinal() == 1
fraction["value"] = 0.5
assert dd.resolve_auto_cuda_ordinal() == 0
_host(monkeypatch, [0, 1], [_dev(0, 80.0, 0.6), _dev(1, 80.0, 0.5)])
assert dd.resolve_auto_cuda_ordinal() is None
def test_route_ranks_the_next_load_under_its_own_budget(monkeypatch):
from core.inference import diffusion_memory as dm
from routes import inference as routes
from utils import vram_budget_settings
fraction = {"value": vram_budget_settings.VRAM_FRACTION_DEFAULT}
monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: fraction["value"])
monkeypatch.setattr(dm, "_LOAD_BUDGET_FRACTIONS", {})
_host(monkeypatch, [0, 1], [_dev(0, 24.0, 1.0), _dev(1, 192.0, 132.0)])
monkeypatch.setattr(
dd, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = "cuda")
)
monkeypatch.setattr(dd, "auto_cuda_ranking_possible_cached", lambda: True)
dm.vram_budget_scope("image", load = True)(lambda: dm.commit_load_budget_fraction("image"))()
fraction["value"] = 0.5
assert dm.vram_budget_scope("image")(dm._vram_budget_fraction)() == pytest.approx(0.97)
seen = {}
@dm.vram_budget_scope("image", load = True)
def plan():
seen["load"] = dm._vram_budget_fraction()
def worker():
seen["worker"] = dm._vram_budget_fraction()
async def request():
ordinal = await routes._selected_gpu_ordinal(None)
fraction["value"] = (
0.8 # edited after the ranking: the load still plans with the ranking's value
)
await asyncio.to_thread(plan)
return ordinal
assert asyncio.run(request()) == 0
assert seen == {"load": pytest.approx(0.5)}
assert dm._vram_budget_fraction() == pytest.approx(0.97)
import threading
fraction["value"] = 0.5
carried = dm.vram_budget_scope("image", load = True)(
lambda: (fraction.update(value = 0.8), dm.carry_load_budget(worker))[1]
)()
thread = threading.Thread(target = carried)
thread.start()
thread.join()
assert seen["worker"] == pytest.approx(0.5)