Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: size the popular audio.cpp TTS and music models from measured peaks on each runtime backend (#16984)

oobabooga115fbb20f6dc
diff --git a/studio/backend/core/inference/audio_cpp_failures.py b/studio/backend/core/inference/audio_cpp_failures.py
index 58f99981bf6..c32b8b04095 100644
--- a/studio/backend/core/inference/audio_cpp_failures.py
+++ b/studio/backend/core/inference/audio_cpp_failures.py
@@ -134,31 +134,67 @@ def attach_failure(exc: BaseException, failure: Optional[AudioCppFailure]) -> Ba
return exc
-# GiB; resident = weights * per_weight + fixed. Generic matches Qwen3-TTS at its 8192-token cap.
+# GiB; resident = weights * per_weight + fixed. The generic lines are per-backend fallbacks for
+# unmeasured families; a lower CUDA line would run a held-out measured family out of memory.
_GENERIC_PROFILE = (2.5, 1.5)
-# family -> (per_weight, fixed, largest package measured): audio.cpp v0.9.3 peak plus at least
-# 0.9 GiB. Families whose peak grows with the input (VibeVoice) must stay on the generic formula.
+_VULKAN_GENERIC_PROFILE = (1.85, 4.5)
+# Peak keeps growing with the input, so no line fit to peaks holds: the CUDA fallback everywhere.
+_UNBOUNDED_FAMILIES = frozenset({"vibevoice"})
+# family -> (per_weight, fixed, largest package measured): audio.cpp v0.9.3 CUDA peak plus at least
+# 0.9 GiB, at default options and each family's generation cap; a raised token cap can exceed it.
_MEASURED_PROFILES = {
- # Peak is a 240 s cover (the longest edit), above a 240 s song; BF16 highest on Vulkan.
+ "ace_step": (0.98, 6.46, 9.4),
+ "breeze_tts": (0.55, 5.18, 6.9),
+ # Covers convert sources up to 30 s; a 60 s source peaks at 12.7 GiB.
+ "chatterbox": (0.05, 5.98, 3.5),
+ "cosyvoice3": (0.6, 2.82, 6.6),
+ "heartmula": (2.5, 1.48, 7.2),
+ # Measured at the default 2048-token chunk cap; a higher max_tokens grows the cache.
+ "higgs_audio_tts": (1.0, 2.4, 8.0),
+ # Also covers the 2.74 GiB #16514 measured for Q8_0 (whole device).
+ "kokoro_tts": (0.0, 3.65, 0.9),
+ "midashenglm_gen": (1.0, 1.69, 3.2),
+ "minimax_music3": (0.7, 8.49, 12.7),
+ "moss_tts_v15": (0.95, 0.74, 19.2),
+ "omnivoice": (0.05, 5.29, 1.6),
+ # Steeper than the tightest fit so 0.6B Q8_0 (7.2 GB) stays loadable on an 8 GB card.
+ "qwen3_tts": (1.5, 4.44, 4.3),
+ "stable_audio": (1.5, 1.17, 5.5),
+ # Kept from #16514 (whole-device peaks above these runs); also covers Vulkan, so no entry there.
+ "voxcpm2": (2.2, 4.7, 8.9),
+ "voxtral_tts": (1.0, 2.0, 7.5),
+ "yue2": (1.03, 4.88, 7.1),
+}
+# The same inputs on audio.cpp's Vulkan build (AMD hosts); a family absent here uses its CUDA line.
+_VULKAN_PROFILES = {
"ace_step": (1.15, 5.55, 9.4),
- "chatterbox": (1.0, 3.5, 3.5),
- # Peak is a 240 s song with 20,000 characters of lyrics, above the generic formula on Vulkan.
+ "breeze_tts": (0.0, 12.1, 6.9),
+ "chatterbox": (0.25, 5.25, 3.5),
+ "cosyvoice3": (0.6, 4.38, 6.6),
"heartmula": (2.5, 2.7, 7.2),
- "higgs_audio_tts": (1.0, 2.0, 4.8),
- "kokoro_tts": (1.0, 3.5, 0.2),
- # Flat in length: 10 s and the 80 s cap peak the same.
+ "higgs_audio_tts": (0.95, 2.94, 8.0),
+ "kokoro_tts": (0.1, 3.89, 0.9),
"midashenglm_gen": (1.0, 2.25, 3.2),
- # Peak is 4 batched takes at the cap (Small 120 s, Medium 240 s); F16 highest on Vulkan.
+ "minimax_music3": (0.7, 9.41, 12.7),
+ "moss_tts_v15": (1.0, 2.63, 19.2),
+ "omnivoice": (0.05, 7.67, 1.6),
+ "qwen3_tts": (1.1, 8.87, 4.3),
"stable_audio": (1.8, 0.65, 5.5),
- "voxcpm2": (2.2, 4.7, 4.5),
- "voxtral_tts": (1.0, 2.0, 4.4),
- # Peak is 20,000 characters of lyrics run to the 9000-token (360 s) cap, highest on Vulkan.
+ "voxtral_tts": (1.7, -0.64, 7.5),
"yue2": (1.2, 4.45, 7.1),
}
# Families measured far higher on pre-Ampere NVIDIA cards (YuE2 on a T4): the generic formula there.
_NOT_BEFORE_AMPERE = frozenset({"yue2"})
+def _runtime_backend() -> Optional[str]:
+ """The installed audio.cpp's backend ("cuda", "vulkan", ...), or None when none is installed."""
+ from core.inference.audio_cpp_server import find_audio_cpp_server_binary, select_backend
+
+ binary = find_audio_cpp_server_binary()
+ return select_backend(binary, False) if binary else None
+
+
def _may_have_pre_ampere_nvidia_card() -> bool:
"""Uncached (gpu_query caches); unreadable NVIDIA capabilities count as pre-Ampere."""
from utils.hardware import nvidia
@@ -187,12 +223,24 @@ def estimate_resident_gb(size_bytes: int, family: Optional[str] = None) -> float
A measured family uses its own profile; a package past the sizes measured for it gets the
larger of that profile and the generic one."""
weights_gb = size_bytes / (1024**3)
- generic = weights_gb * _GENERIC_PROFILE[0] + _GENERIC_PROFILE[1]
+ backend = _runtime_backend()
+
+ def pick(cuda, vulkan):
+ # Not installed yet, or a build measured on neither (HIP, CPU): both, and the larger wins.
+ return {"cuda": [cuda], "vulkan": [vulkan]}.get(backend, [cuda, vulkan])
+
+ generics = (
+ [_GENERIC_PROFILE]
+ if family in _UNBOUNDED_FAMILIES
+ else pick(_GENERIC_PROFILE, _VULKAN_GENERIC_PROFILE)
+ )
+ generic = max(weights_gb * per_weight + fixed for per_weight, fixed in generics)
profile = _MEASURED_PROFILES.get(family or "")
if profile is None or (family in _NOT_BEFORE_AMPERE and _may_have_pre_ampere_nvidia_card()):
return round(generic, 2)
- per_weight, fixed, measured_up_to_gb = profile
- measured = weights_gb * per_weight + fixed
+ profiles = pick(profile, _VULKAN_PROFILES[family]) if family in _VULKAN_PROFILES else [profile]
+ measured = max(weights_gb * per_weight + fixed for per_weight, fixed, _largest in profiles)
+ measured_up_to_gb = min(largest for _per_weight, _fixed, largest in profiles)
return round(measured if weights_gb <= measured_up_to_gb else max(measured, generic), 2)
diff --git a/studio/backend/tests/test_audio_cpp_failure_routes.py b/studio/backend/tests/test_audio_cpp_failure_routes.py
index ea5aa1ac562..fd1d99501b0 100644
--- a/studio/backend/tests/test_audio_cpp_failure_routes.py
+++ b/studio/backend/tests/test_audio_cpp_failure_routes.py
@@ -22,6 +22,12 @@ from core.inference import audio_cpp_server as srv
from models.inference import ChatCompletionRequest, LoadRequest
from .test_audio_cpp_failures import _model
+
[email protected](autouse = True)
+def _cuda_audio_runtime(monkeypatch):
+ monkeypatch.setattr(af, "_runtime_backend", lambda: "cuda")
+
+
_MISSING = "The audio runtime is not installed. Run `unsloth studio update` to install it."
@@ -162,11 +168,11 @@ def test_with_nothing_to_unload_the_refusal_reads_the_cards_free_memory(monkeypa
# 7.3 GB free is an 8 GB card under a desktop session.
("voxtral_tts", 3, 7.3, True),
("chatterbox", 3, 7.3, True),
- ("qwen3_tts", 3, 7.3, False),
+ ("neutts", 3, 7.3, False),
("vibevoice", 3, 7.3, False),
# Families whose measured peak is above the generic formula are refused where it fit.
("voxcpm2", 2.75, 9.5, False),
- ("qwen3_tts", 2.75, 9.5, True),
+ ("neutts", 2.75, 9.5, True),
("kokoro_tts", 0.18, 3.0, False),
# ACE-Step1.5 Q8_0 on a 16 GB card: measured 11.2 GB, the generic formula said 15.9.
("ace_step", 5.76, 15.0, True),
@@ -194,18 +200,23 @@ def test_placement_uses_the_family_estimate(monkeypatch, family, size_gb, free_g
@pytest.mark.parametrize(
- "capabilities, driver, free_gb, fits",
+ "backend, capabilities, driver, free_gb, required",
[
- ({0: (8, 6)}, True, 9.6, True),
- ({}, False, 9.6, True),
- ({0: (8, 6)}, True, 9.4, False),
- ({0: (7, 5)}, True, 9.6, False),
- ({}, True, 9.6, False),
+ # #16627: a 10 GB card with about 9.4 GiB free.
+ ("cuda", {0: (8, 6)}, True, 9.4, 9.23),
+ ("cuda", {0: (8, 6)}, True, 9.2, None),
+ ("vulkan", {0: (8, 6)}, True, 9.6, 9.51),
+ ("vulkan", {0: (8, 6)}, True, 9.4, None),
+ (None, {}, False, 9.6, 9.51),
+ (None, {0: (8, 6)}, True, 9.4, None),
+ ("cuda", {0: (7, 5)}, True, 9.6, None),
+ ("cuda", {}, True, 9.6, None),
],
)
-def test_yue2_q8_on_a_10gb_card(monkeypatch, capabilities, driver, free_gb, fits):
+def test_yue2_q8_on_a_10gb_card(monkeypatch, backend, capabilities, driver, free_gb, required):
from utils.hardware import nvidia
+ monkeypatch.setattr(af, "_runtime_backend", lambda: backend)
monkeypatch.setattr(af, "offload_options", lambda model: [])
monkeypatch.setattr(nvidia, "get_compute_capabilities", lambda: dict(capabilities))
monkeypatch.setattr(nvidia, "has_nvidia_driver", lambda: driver)
@@ -216,13 +227,13 @@ def test_yue2_q8_on_a_10gb_card(monkeypatch, capabilities, driver, free_gb, fits
model = dataclasses.replace(base, family = "yue2", variant = variant, variants = (variant,))
availability = ri._NativeAudioAvailability({0: free_gb}, {0: 0.0})
prepared = ([0], {"selection_mode": "fallback_all"})
- if not fits:
+ if required is None:
with pytest.raises(HTTPException) as raised:
_preflight(None, availability = availability, prepared = prepared, model = model)
assert raised.value.detail["code"] == "out_of_memory"
return
placement = _preflight(None, availability = availability, prepared = prepared, model = model)
- assert placement.resolved_gpu_ids == [0] and placement.native_required_gb == 9.51
+ assert placement.resolved_gpu_ids == [0] and placement.native_required_gb == required
def test_generation_crash_oom_maps_to_out_of_memory(monkeypatch):
diff --git a/studio/backend/tests/test_audio_cpp_failures.py b/studio/backend/tests/test_audio_cpp_failures.py
index 6ad5d08d4a5..bd131908c4e 100644
--- a/studio/backend/tests/test_audio_cpp_failures.py
+++ b/studio/backend/tests/test_audio_cpp_failures.py
@@ -13,6 +13,13 @@ import pytest
from core.inference import audio_cpp_failures as af
from core.inference.audio_cpp_models import AudioCppModel, AudioCppVariant, RepoFile
+_REAL_RUNTIME_BACKEND = af._runtime_backend
+
+
[email protected](autouse = True)
+def _cuda_audio_runtime(monkeypatch):
+ monkeypatch.setattr(af, "_runtime_backend", lambda: "cuda")
+
def _model(sizes = (8, 4, 3, 2, 1), current = 0) -> AudioCppModel:
variants = tuple(
@@ -24,7 +31,7 @@ def _model(sizes = (8, 4, 3, 2, 1), current = 0) -> AudioCppModel:
repo_id = "audio-cpp/audio.cpp-gguf",
folder = "M-GGUF",
display_name = "Model M",
- family = "qwen3_tts",
+ family = "neutts",
task = "tts",
server_task = "tts",
variant = variants[current],
@@ -123,46 +130,144 @@ def test_studio_runtime_messages_classify(tmp_path, monkeypatch):
assert newer.code == "family_unknown_to_runtime" and "Model M" in newer.message
-def test_estimate_uses_the_measured_family_profile():
+def test_estimate_uses_the_measured_family_profile(monkeypatch):
+ monkeypatch.setattr(af, "_runtime_backend", lambda: "cuda")
gb = 1024**3
generic = af.estimate_resident_gb(3 * gb)
- assert generic == af.estimate_resident_gb(3 * gb, "qwen3_tts") == 9.0
+ assert generic == af.estimate_resident_gb(3 * gb, "neutts") == 9.0
assert af.estimate_resident_gb(3 * gb, None) == generic
assert af.estimate_resident_gb(3 * gb, "new") == generic
# Peak grows with the input, so it keeps the generic formula.
assert af.estimate_resident_gb(3 * gb, "vibevoice") == generic
# Measured below the generic formula.
- assert af.estimate_resident_gb(3 * gb, "chatterbox") == 6.5
+ assert af.estimate_resident_gb(3 * gb, "chatterbox") == 6.13
assert af.estimate_resident_gb(int(4.4 * gb), "voxtral_tts") == 6.4
- # Measured above it: Kokoro q8 peaks at 2.7 GB, VoxCPM2 q8 at 9.7 GB and bf16 at 13.3 GB.
- assert af.estimate_resident_gb(int(0.1765 * gb), "kokoro_tts") == 3.68 > 2.7
- assert af.estimate_resident_gb(int(2.7521 * gb), "voxcpm2") == 10.75 > 9.7
- assert af.estimate_resident_gb(int(4.4445 * gb), "voxcpm2") == 14.48 > 13.3
- # ACE-Step1.5 turbo Q8_0 / BF16 (Hub bytes): a 240 s cover peaks at 11.22 / 15.31 GiB.
- assert af.estimate_resident_gb(6185468096, "ace_step") == 12.17 > 11.22 + 0.9
- assert af.estimate_resident_gb(10090406336, "ace_step") == 16.36 > 15.31 + 0.9
+ # Measured above it (#16514, whole device): Kokoro q8 2.74 GiB, VoxCPM2 q8 9.66 and bf16 13.31.
+ assert af.estimate_resident_gb(int(0.1765 * gb), "kokoro_tts") == 3.65 >= 2.74 + 0.9
+ assert af.estimate_resident_gb(int(2.7521 * gb), "voxcpm2") == 10.75 >= 9.66 + 0.9
+ assert af.estimate_resident_gb(int(4.4445 * gb), "voxcpm2") == 14.48 >= 13.31 + 0.9
+ # ACE-Step1.5 turbo Q8_0 / BF16 (Hub bytes): a 240 s cover peaks at 11.19 / 14.74 GiB on CUDA.
+ assert af.estimate_resident_gb(6185468096, "ace_step") == 12.11 > 11.19 + 0.9
+ assert af.estimate_resident_gb(10090406336, "ace_step") == 15.67 > 14.74 + 0.9
+
+
+GIB = 1024**3
+# family, package bytes, worst CUDA peak, worst Vulkan peak (GiB) for every measured variant.
+MEASURED_PEAKS = [
+ ("ace_step", 6185468096, 11.19, 11.22),
+ ("ace_step", 10090406336, 14.74, 15.31),
+ ("breeze_tts", int(3.534 * GIB), 5.5, 6.83),
+ ("breeze_tts", int(4.153 * GIB), 6.12, 8.39),
+ ("breeze_tts", int(4.731 * GIB), 6.87, 11.18),
+ ("breeze_tts", int(6.839 * GIB), 8.0, 9.79),
+ ("chatterbox", int(1.945 * GIB), 5.16, 4.82),
+ ("chatterbox", int(3.487 * GIB), 5.23, 5.19),
+ ("cosyvoice3", int(2.103 * GIB), 3.08, 4.73),
+ ("cosyvoice3", int(6.515 * GIB), 5.81, 7.36),
+ ("heartmula", 7659762592, 18.4, 19.58),
+ ("higgs_audio_tts", int(4.745 * GIB), 6.19, 6.47),
+ ("higgs_audio_tts", int(7.918 * GIB), 9.4, 9.55),
+ ("kokoro_tts", int(0.177 * GIB), 2.41, 2.99),
+ ("kokoro_tts", int(0.197 * GIB), 2.39, 2.12),
+ ("kokoro_tts", int(0.869 * GIB), 2.41, 3.05),
+ ("midashenglm_gen", 3381370752, 3.93, 4.42),
+ ("minimax_music3", int(7.866 * GIB), 13.08, 14.0),
+ ("minimax_music3", int(12.609 * GIB), 16.35, 17.26),
+ ("moss_tts_v15", int(8.622 * GIB), 7.79, 10.34),
+ ("moss_tts_v15", int(12.279 * GIB), 11.25, 12.6),
+ ("moss_tts_v15", int(19.134 * GIB), 18.0, 20.58),
+ ("omnivoice", int(1.258 * GIB), 4.44, 6.82),
+ ("omnivoice", int(1.527 * GIB), 3.96, 4.92),
+ ("qwen3_tts", int(1.854 * GIB), 5.88, 9.18),
+ ("qwen3_tts", int(2.343 * GIB), 6.36, 9.66),
+ ("qwen3_tts", int(2.510 * GIB), 6.73, 9.5),
+ ("qwen3_tts", int(2.624 * GIB), 7.47, 10.84),
+ ("qwen3_tts", int(3.892 * GIB), 8.85, 12.2),
+ ("qwen3_tts", int(3.915 * GIB), 8.11, 12.06),
+ ("qwen3_tts", int(4.232 * GIB), 8.2, 11.49),
+ ("stable_audio", 1683576608, 2.24, 2.01),
+ ("stable_audio", 2358473088, 2.8, 2.54),
+ ("stable_audio", 3582601408, 5.27, 5.34),
+ ("stable_audio", 5834365856, 7.3, 9.48),
+ ("voxcpm2", int(2.752 * GIB), 8.62, 8.94),
+ ("voxcpm2", int(4.445 * GIB), 11.09, 11.15),
+ ("voxcpm2", int(4.620 * GIB), 11.09, 13.11),
+ ("voxcpm2", int(8.889 * GIB), 19.06, 22.36),
+ ("voxtral_tts", int(4.399 * GIB), 5.31, 5.92),
+ ("voxtral_tts", int(7.479 * GIB), 8.38, 11.15),
+ ("yue2", 2933424181, 6.78, 6.52),
+ ("yue2", 4531978293, 8.27, 8.58),
+ ("yue2", 7529267253, 11.19, 11.87),
+]
+
+
[email protected]("family, size_bytes, cuda_peak, vulkan_peak", MEASURED_PEAKS)
[email protected]("backend", ["cuda", "vulkan"])
+def test_each_runtime_profile_covers_its_measured_peak(
+ monkeypatch, family, size_bytes, cuda_peak, vulkan_peak, backend
+):
+ monkeypatch.setattr(af, "_runtime_backend", lambda: backend)
+ monkeypatch.setattr(af, "_may_have_pre_ampere_nvidia_card", lambda: False)
+ peak = cuda_peak if backend == "cuda" else vulkan_peak
+ assert af.estimate_resident_gb(size_bytes, family) >= peak + 0.9
@pytest.mark.parametrize(
- "family, size_bytes, peak_gib, expected",
+ "family, size_bytes, cuda, vulkan",
[
- ("stable_audio", 1683576608, 2.24, 3.47),
- ("stable_audio", 2358473088, 2.80, 4.60),
- ("stable_audio", 3582601408, 5.34, 6.66),
- ("stable_audio", 5834365856, 9.48, 10.43),
- ("midashenglm_gen", 3381370752, 4.42, 5.40),
- ("heartmula", 7659762592, 19.58, 20.53),
+ # #16627's 10 GB card (about 9.4 GiB free) fits these two on CUDA.
+ ("yue2", 4531978293, 9.23, 9.51),
+ ("stable_audio", 5834365856, 9.32, 10.43),
+ ("qwen3_tts", int(1.854 * GIB), 7.22, 10.91),
+ ("chatterbox", int(1.945 * GIB), 6.08, 5.74),
],
)
-def test_music_profiles_cover_their_measured_peak(family, size_bytes, peak_gib, expected):
- estimate = af.estimate_resident_gb(size_bytes, family)
- assert estimate == expected and estimate >= peak_gib + 0.9
+def test_the_runtime_backend_picks_the_profile(monkeypatch, family, size_bytes, cuda, vulkan):
+ monkeypatch.setattr(af, "_may_have_pre_ampere_nvidia_card", lambda: False)
+ either = max(cuda, vulkan)
+ for backend, expected in (("cuda", cuda), ("vulkan", vulkan), (None, either), ("hip", either)):
+ monkeypatch.setattr(af, "_runtime_backend", lambda b = backend: b)
+ assert af.estimate_resident_gb(size_bytes, family) == expected, backend
+
+
+ "backend, at_3, at_9_5",
+ [
+ ("cuda", 9.0, 25.25),
+ ("vulkan", 10.05, 22.07),
+ (None, 10.05, 25.25),
+ ("hip", 10.05, 25.25),
+ ],
+)
+def test_the_fallback_follows_the_runtime_backend(monkeypatch, backend, at_3, at_9_5):
+ monkeypatch.setattr(af, "_runtime_backend", lambda: backend)
+ gb = 1024**3
+ assert af.estimate_resident_gb(3 * gb, "neutts") == at_3
+ assert af.estimate_resident_gb(int(9.5 * gb), "neutts") == at_9_5
+ assert af.estimate_resident_gb(3 * gb, "vibevoice") == 9.0
+ assert af.estimate_resident_gb(int(9.85 * gb), "vibevoice") == 26.12
+
+
+def test_the_runtime_backend_is_the_installed_servers(monkeypatch):
+ from core.inference import audio_cpp_server as srv
+
+ monkeypatch.setattr(srv, "find_audio_cpp_server_binary", lambda: None)
+ assert _REAL_RUNTIME_BACKEND() is None
+ monkeypatch.setattr(srv, "find_audio_cpp_server_binary", lambda: "/opt/audiocpp_server")
+ monkeypatch.setattr(
+ srv, "select_backend", lambda binary, force_cpu: f"vulkan:{binary}:{force_cpu}"
+ )
+ assert _REAL_RUNTIME_BACKEND() == "vulkan:/opt/audiocpp_server:False"
# YuE2 package bytes on the Hub (model + VAE + sidecars): Q4_0, Q8_0, BF16.
YUE2_BYTES = (2933424181, 4531978293, 7529267253)
+# MEASURED is the Vulkan profile, which also wins with no runtime installed.
MEASURED = (7.73, 9.51, 12.86)
+MEASURED_CUDA = (7.69, 9.23, 12.1)
GENERIC = (8.33, 12.05, 19.03)
+GENERIC_VULKAN = (9.55, 12.31, 17.47)
+GENERIC_EITHER = (9.55, 12.31, 19.03)
def _cards(
@@ -182,22 +287,31 @@ def _yue2():
@pytest.mark.parametrize(
- "capabilities, driver, expected",
+ "capabilities, driver, profiled",
+ [
+ ({0: (8, 6)}, True, True),
+ ({0: (8, 9)}, True, True),
+ ({0: (12, 0)}, True, True),
+ ({}, False, True),
+ ({0: (7, 5)}, True, False),
+ ({0: (8, 9), 1: (7, 5)}, True, False),
+ ({}, True, False),
+ ],
+)
+ "backend, measured, generic",
[
- ({0: (8, 6)}, True, MEASURED),
- ({0: (8, 9)}, True, MEASURED),
- ({0: (12, 0)}, True, MEASURED),
- ({}, False, MEASURED),
- ({0: (7, 5)}, True, GENERIC),
- ({0: (8, 9), 1: (7, 5)}, True, GENERIC),
- ({}, True, GENERIC),
+ (None, MEASURED, GENERIC_EITHER),
+ ("vulkan", MEASURED, GENERIC_VULKAN),
+ ("cuda", MEASURED_CUDA, GENERIC),
],
)
def test_yue2_uses_its_profile_unless_a_pre_ampere_nvidia_card_may_be_present(
- monkeypatch, capabilities, driver, expected
+ monkeypatch, capabilities, driver, profiled, backend, measured, generic
):
+ monkeypatch.setattr(af, "_runtime_backend", lambda: backend)
_cards(monkeypatch, capabilities, driver = driver)
- assert _yue2() == expected
+ assert _yue2() == (measured if profiled else generic)
@pytest.mark.parametrize(
@@ -214,7 +328,7 @@ def test_an_nvidia_host_is_told_from_one_without_nvidia(monkeypatch, smi, procfs
def test_the_yue2_estimate_follows_the_current_inventory(monkeypatch):
_cards(monkeypatch, {0: (8, 6)}, {0: (8, 6), 1: (7, 5)}, {0: (8, 6)})
- assert (_yue2(), _yue2(), _yue2()) == (MEASURED, GENERIC, MEASURED)
+ assert (_yue2(), _yue2(), _yue2()) == (MEASURED_CUDA, GENERIC, MEASURED_CUDA)
def test_past_the_measured_sizes_the_estimate_never_drops_below_either_profile():
@@ -222,14 +336,15 @@ def test_past_the_measured_sizes_the_estimate_never_drops_below_either_profile()
# A bigger model of a family measured low: the generic formula.
assert af.estimate_resident_gb(8 * gb, "voxtral_tts") == af.estimate_resident_gb(8 * gb) == 21.5
# A bigger model of a family measured high: its own profile.
- assert af.estimate_resident_gb(int(8.8887 * gb), "voxcpm2") == 24.26
- assert af.estimate_resident_gb(int(8.8887 * gb)) == 23.72
+ assert af.estimate_resident_gb(int(9.5 * gb), "voxcpm2") == 25.6
+ assert af.estimate_resident_gb(int(9.5 * gb)) == 25.25
def test_smaller_variants_estimate_with_the_family(monkeypatch):
- model = replace(_model(sizes = (6, 3, 1), current = 0), family = "chatterbox")
+ monkeypatch.setattr(af, "_runtime_backend", lambda: "cuda")
+ model = replace(_model(sizes = (6, 3, 1), current = 0), family = "higgs_audio_tts")
rows = af.smaller_variants(model)
- assert [(row["label"], row["estimate_gb"]) for row in rows] == [("Q3", 6.5), ("Q1", 4.5)]
+ assert [(row["label"], row["estimate_gb"]) for row in rows] == [("Q3", 5.4), ("Q1", 3.4)]
assert [row["label"] for row in af.smaller_variants(model, free_gb = 7.3)] == ["Q3", "Q1"]
diff --git a/studio/backend/tests/test_audio_cpp_models.py b/studio/backend/tests/test_audio_cpp_models.py
index 7eb0d926faf..034896f66e6 100644
--- a/studio/backend/tests/test_audio_cpp_models.py
+++ b/studio/backend/tests/test_audio_cpp_models.py
@@ -1881,9 +1881,11 @@ def test_umbrella_repo_is_hidden_from_chat():
assert is_hidden_model("Audio-CPP/audio.cpp-GGUF")
-def test_resident_estimate_scales_with_the_variant():
+def test_resident_estimate_scales_with_the_variant(monkeypatch):
import routes.inference as ri
+ from core.inference import audio_cpp_failures as af
+ monkeypatch.setattr(af, "_runtime_backend", lambda: "cuda")
small = ri.audio_cpp_resident_gb(237 * 1024**2)
big = ri.audio_cpp_resident_gb(5899 * 1024**2)
assert 1.5 < small < 3 and 15 < big < 18