Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let YuE2 songs run to their own ending instead of 1.5x the length (#13371)

Daniel Han1d79d47c464e
diff --git a/studio/backend/core/inference/audio_cpp_backend.py b/studio/backend/core/inference/audio_cpp_backend.py
index 4a49bd4326d..56d46218163 100644
--- a/studio/backend/core/inference/audio_cpp_backend.py
+++ b/studio/backend/core/inference/audio_cpp_backend.py
@@ -889,10 +889,11 @@ class AudioCppBackend:
)
except cm.MusicRequestError as exc:
raise RuntimeError(str(exc)) from exc
+ own_end = cm.self_ending_timeout(model.family, server.backend)
ctype, data = server.post_json(
"/v1/tasks/run",
{"model": server.model_id, "request": request},
- timeout = _GENERATE_TIMEOUT_SECONDS,
+ timeout = own_end or _GENERATE_TIMEOUT_SECONDS,
cancel_event = cancel_event,
)
return _audio_from_task_response(ctype, data)
diff --git a/studio/backend/core/inference/audio_cpp_models.py b/studio/backend/core/inference/audio_cpp_models.py
index cae82fadc2c..ab531c59b8d 100644
--- a/studio/backend/core/inference/audio_cpp_models.py
+++ b/studio/backend/core/inference/audio_cpp_models.py
@@ -194,8 +194,8 @@ class MusicSpec:
# The runtime's default seed is fixed, so repeat runs would be identical.
fixed_seed: bool = True
rtf: float = 4.0
- # Token budget as a multiple of the asked length, for a model that ends the song itself.
- overrun: float = 1.0
+ # A model that ends the song itself gets no length budget; the runtime caps it at this many s.
+ self_ending_s: Optional[float] = None
def mode(self, mode_id: Optional[str]) -> Optional[MusicMode]:
return next((m for m in self.modes if m.id == mode_id), None)
@@ -265,8 +265,8 @@ MUSIC_SPECS: dict[str, MusicSpec] = {
# Empty lyrics still sing; only this tag stays wordless.
instrumental_lyrics = "[Instrumental]",
rtf = 6.0,
- # 1.0x cut long lyrics mid-song (#13337); 240 s x 1.5 = the runtime's 9000-token default.
- overrun = 1.5,
+ # Any length budget cut songs mid-verse (#13337); upstream's own cap is 9000 frames = 360 s.
+ self_ending_s = 360.0,
),
}
diff --git a/studio/backend/core/inference/audio_cpp_music.py b/studio/backend/core/inference/audio_cpp_music.py
index 06d6b3b441b..61563c4992d 100644
--- a/studio/backend/core/inference/audio_cpp_music.py
+++ b/studio/backend/core/inference/audio_cpp_music.py
@@ -55,12 +55,22 @@ def timeout_seconds(
family: Optional[str], seconds: float, variations: int, backend: Optional[str]
) -> float:
music = MUSIC_SPECS.get(str(family or ""))
- rtf = music.rtf * music.overrun if music is not None else 4.0
+ rtf = music.rtf if music is not None else 4.0
rtf *= _BACKEND_FACTOR.get(str(backend or "").lower(), 1.0)
+ if music is not None and music.self_ending_s:
+ seconds = music.self_ending_s
work = rtf * max(0.0, float(seconds)) * max(1, int(variations))
return max(_TIMEOUT_FLOOR, min(_TIMEOUT_CEILING, _TIMEOUT_FLOOR + work))
+def self_ending_timeout(family: Optional[str], backend: Optional[str]) -> Optional[float]:
+ """The wait for a model that ends the song itself, whatever length or token count was asked."""
+ music = MUSIC_SPECS.get(str(family or ""))
+ if music is None or not music.self_ending_s:
+ return None
+ return timeout_seconds(family, music.self_ending_s, 1, backend)
+
+
def frames_for(seconds: float, variations: int = 1) -> int:
return int(math.ceil(max(0.0, float(seconds)) * 25 * max(1, int(variations))))
@@ -196,11 +206,6 @@ def song_request(
request["text"] = lyrics
request_options["style"] = description
request_options["lyrics"] = lyrics
- # Length caps the semantic tokens (25 fps); the default 200-frame floor outlasts short asks.
- frames = int(round(seconds * 25))
- overrun = MUSIC_SPECS["yue2"].overrun
- request_options["semantic_max_tokens"] = int(round(seconds * overrun * 25))
- request_options["semantic_min_tokens"] = min(200, frames)
return _finish(model, request, request_options, seed)
if instrumental and music is not None and music.instrumental_lyrics is not None:
lyrics = music.instrumental_lyrics
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 5604869e83b..287b706cb7c 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -244,6 +244,25 @@ def _audio_generation_timeout(
return base * token_scale
+def _music_generation_timeout(
+ timeout: float, audio_type: Optional[str], model_info: dict, music: Optional[dict]
+) -> float:
+ """Outlast the worker's music wait so its error, not the watchdog, reaches the caller."""
+ if music is not None:
+ try:
+ wait = float(music.get("timeout_s") or 0.0)
+ except (TypeError, ValueError):
+ wait = 0.0
+ elif audio_type == "audiocpp_music":
+ # Legacy calls carry no wait; a self-ending model (YuE2) runs past any token count.
+ from core.inference.audio_cpp_music import self_ending_timeout
+ backend = model_info.get("audio_cpp_backend")
+ wait = self_ending_timeout(model_info.get("audio_family"), backend) or 0.0
+ else:
+ return timeout
+ return max(timeout, wait + 60.0)
+
+
_MLX_RUNTIME_MIRROR_FIELDS = (
"mlx_kv_bits",
"mlx_kv_bits_requested",
@@ -3782,12 +3801,9 @@ class InferenceOrchestrator:
cmd["edit"] = dict(edit)
if music is not None:
cmd["music"] = dict(music)
- try:
- music_wait = float(music.get("timeout_s") or 0.0)
- except (TypeError, ValueError):
- music_wait = 0.0
- # Outlast the worker's wait so its error, not the watchdog, reaches the caller.
- generation_timeout = max(generation_timeout, music_wait + 60.0)
+ generation_timeout = _music_generation_timeout(
+ generation_timeout, audio_type, model_info, music
+ )
if output_dir is not None:
cmd["output_dir"] = str(output_dir)
diff --git a/studio/backend/tests/test_audio_cpp_models.py b/studio/backend/tests/test_audio_cpp_models.py
index 1dfecd9d50b..83d18a4109e 100644
--- a/studio/backend/tests/test_audio_cpp_models.py
+++ b/studio/backend/tests/test_audio_cpp_models.py
@@ -17,7 +17,7 @@ from types import SimpleNamespace
import pytest
-from core.inference import audio_cpp_backend, audio_cpp_files
+from core.inference import audio_cpp_backend, audio_cpp_files, audio_cpp_music
from core.inference import audio_cpp_models as acm
from core.inference.audio_cpp_models import (
AUDIO_CPP_REPO,
@@ -1618,12 +1618,7 @@ def test_yue2_takes_the_description_as_its_required_style():
b.generate_audio_response("[chorus] oh", instructions = "rock, male vocal")
request = srv.calls[0][1]["request"]
assert request["text"] == "[chorus] oh"
- assert request["options"] == {
- "style": "rock, male vocal",
- "lyrics": "[chorus] oh",
- "semantic_max_tokens": 3072,
- "semantic_min_tokens": 200,
- }
+ assert request["options"] == {"style": "rock, male vocal", "lyrics": "[chorus] oh"}
with pytest.raises(RuntimeError, match = "style"):
b.generate_audio_response("[chorus] oh")
@@ -2624,17 +2619,21 @@ def test_stt_takes_the_variant_keys_gguf_variants_lists_even_from_a_partial_cach
assert stem.variant.key == "Q8_0" and stem.unsupported is None
-def test_yue2_length_follows_the_requested_duration():
+def test_yue2_runs_to_its_own_ending_whatever_the_token_budget():
b, srv = _backend_with(_speech_model("yue2"), MUSIC_REPLY)
- b.generate_audio_response("[chorus] oh", instructions = "rock", max_new_tokens = 125)
- options = srv.calls[0][1]["request"]["options"]
- # 5 s at 25 frames per second, with the 200-frame default lowered to fit; the cap is 1.5x.
- assert options["semantic_max_tokens"] == 188 and options["semantic_min_tokens"] == 125
- b.generate_audio_response("[chorus] oh", instructions = "rock", max_new_tokens = 1500)
- options = srv.calls[1][1]["request"]["options"]
- assert options["semantic_max_tokens"] == 2250 and options["semantic_min_tokens"] == 200
- b.generate_audio_response("[chorus] oh", instructions = "rock", max_new_tokens = 6000)
- assert srv.calls[2][1]["request"]["options"]["semantic_max_tokens"] == 9000
+ srv.backend = "cpu"
+ waits = []
+ post = srv.post_json
+ srv.post_json = lambda path, payload, **kw: waits.append(kw["timeout"]) or post(path, payload)
+ for budget in (125, 1500, 6000):
+ b.generate_audio_response("[chorus] oh", instructions = "rock", max_new_tokens = budget)
+ for _path, body in srv.calls:
+ options = body["request"]["options"]
+ assert "semantic_max_tokens" not in options and "semantic_min_tokens" not in options
+ assert waits == [audio_cpp_music.self_ending_timeout("yue2", "cpu")] * 3 == [4 * 3600.0] * 3
+ srv.backend = "cuda"
+ b.generate_audio_response("[chorus] oh", instructions = "rock", max_new_tokens = 2048)
+ assert waits[-1] == audio_cpp_music.timeout_seconds("yue2", 240, 1, "cuda") == 2460.0
def test_music_requests_missing_a_required_field_are_a_400_message():
diff --git a/studio/backend/tests/test_audio_cpp_music_requests.py b/studio/backend/tests/test_audio_cpp_music_requests.py
index 8ed4ecbb259..22ec0637b62 100644
--- a/studio/backend/tests/test_audio_cpp_music_requests.py
+++ b/studio/backend/tests/test_audio_cpp_music_requests.py
@@ -375,7 +375,16 @@ def test_yue2_instrumental_sends_the_instrumental_tag_not_its_style():
assert request["text"] == "[Instrumental]"
assert request["options"]["lyrics"] == "[Instrumental]"
assert request["options"]["style"] == "ambient piano"
- assert request["options"]["semantic_max_tokens"] == 750
+ assert "semantic_max_tokens" not in request["options"]
+
+
+def test_yue2_songs_are_not_cut_at_the_asked_length():
+ # Any budget from the slider is a hard stop the lyrics outrun (#13337).
+ for seconds in (5, 30, 240):
+ b = _backend(_model("yue2"))
+ _run(b, text = "rock", lyrics = "[Verse] words", duration_s = seconds)
+ options = _request(b)["options"]
+ assert "semantic_max_tokens" not in options and "semantic_min_tokens" not in options
def test_minimax_gguf_keeps_its_body_through_the_studio():
@@ -494,10 +503,12 @@ def test_a_fixed_seed_family_gets_a_recorded_random_seed(tmp_path):
def test_the_wait_grows_with_the_audio_asked_for_and_is_capped():
t = lambda s, n = 1, backend = "cuda": cm.timeout_seconds("yue2", s, n, backend)
- assert t(10) < t(60) < t(60, 3) < t(60, 3, backend = "cpu")
- assert t(0) == 300.0 and t(240, 4, backend = "cpu") == 4 * 3600.0
- assert cm.timeout_seconds("stable_audio", 10, 1, "cuda") == 310.0
- assert t(60) == 300.0 + 6.0 * 1.5 * 60
+ assert t(60) < t(60, 3) < t(60, 3, backend = "cpu")
+ assert t(240, 4, backend = "cpu") == 4 * 3600.0
+ sa = lambda s: cm.timeout_seconds("stable_audio", s, 1, "cuda")
+ assert sa(0) == 300.0 and sa(10) == 310.0 and sa(10) < sa(60)
+ # YuE2 may run to its 360 s cap whatever length was asked.
+ assert t(0) == t(60) == 300.0 + 6.0 * 360
b = _backend(_model("yue2"))
_run(b, text = "rock", lyrics = "x", duration_s = 60)
assert b._server.calls[0][2]["timeout"] == t(60)
diff --git a/studio/backend/tests/test_audio_tts_cancellation.py b/studio/backend/tests/test_audio_tts_cancellation.py
index 1a1e307bd52..6ce7d20f68e 100644
--- a/studio/backend/tests/test_audio_tts_cancellation.py
+++ b/studio/backend/tests/test_audio_tts_cancellation.py
@@ -421,6 +421,25 @@ def test_audio_generation_timeout_scales_with_requested_tokens(monkeypatch):
assert orchestrator_module._audio_generation_timeout(10**310) == 40.0
+def test_legacy_yue2_watchdog_outlasts_its_own_ending():
+ # YuE2 runs to its own end whatever the token count; the 900 s token watchdog cut it (#13337).
+ from core.inference import audio_cpp_music as cm
+
+ timeout = orchestrator_module._music_generation_timeout
+ yue2 = {"audio_family": "yue2", "audio_cpp_backend": "cuda"}
+ assert (
+ timeout(900.0, "audiocpp_music", yue2, None)
+ == cm.timeout_seconds("yue2", 0, 1, "cuda") + 60
+ )
+ cpu = dict(yue2, audio_cpp_backend = "cpu")
+ assert timeout(900.0, "audiocpp_music", cpu, None) == 4 * 3600.0 + 60
+ stable = {"audio_family": "stable_audio", "audio_cpp_backend": "cuda"}
+ assert timeout(900.0, "audiocpp_music", stable, None) == 900.0
+ assert timeout(900.0, "audiocpp_tts", yue2, None) == 900.0
+ assert timeout(900.0, "audiocpp_music", yue2, {"timeout_s": 1234.0}) == 1294.0
+ assert timeout(900.0, "audiocpp_music", yue2, {"timeout_s": "x"}) == 900.0
+
+
def test_tts_route_bounds_public_token_budget():
payload = ChatCompletionRequest(
messages = [{"role": "user", "content": "hello"}],