Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""The exact requests audio.cpp receives for a voice conversion, per family, on a fake server."""
from __future__ import annotations
import base64
import io
import json
import wave
import pytest
from core.inference import audio_cpp_backend as acb
from core.inference import audio_cpp_convert as acc
from core.inference import audio_cpp_files
from core.inference import audio_cpp_models as acm
from core.inference import audio_cpp_server as srv
from core.inference.audio_cpp_models import (
AUDIO_CPP_REPO,
AudioCppModel,
AudioCppVariant,
RepoFile,
)
from core.inference.audio_errors import AudioRuntimeError
SOURCE = "/srv/accounts/a/audio/inputs/0123.16000.mono.m300.wav"
TARGET = "/srv/accounts/a/audio/inputs/4567.16000.mono.m30.wav"
TRANSCRIPT = "Okay, I'm Cemo and what you just heard wasn't a human voice."
_RVC_OPTIONS = (
{"name": "voice_id", "type": "enum", "values": ["default", "manthos", "chocola", "fraise"]},
{"name": "retrieval_blend", "type": "float", "min": 0.0, "max": 1.0, "default": 0.0},
{"name": "semitone_shift", "type": "int", "default": 0},
{"name": "rms_mix_rate", "type": "float", "default": 0.25},
{"name": "unvoiced_protection", "type": "float", "min": 0.0, "max": 1.0, "default": 0.33},
)
_SEED_VC_OPTIONS = (
{
"name": "route",
"type": "enum",
"values": ["v2_vc", "v1_svc", "v1_whisper_bigvgan_vc", "v1_xlsr_hift_vc"],
},
{"name": "length_adjust", "type": "float", "min": 0.0, "default": 1.0},
{"name": "similarity_guidance_scale", "type": "float", "min": 0.0, "default": 0.7},
{"name": "voice_anonymization", "type": "bool", "default": False},
{"name": "auto_f0_adjust", "type": "bool", "default": False},
{"name": "semitone_shift", "type": "int", "default": 0},
)
_BLANK_OPTION = {
"description": "",
"required": False,
"default": None,
"min": None,
"max": None,
"values": None,
}
def _wav(rate) -> bytes:
buf = io.BytesIO()
with wave.open(buf, "wb") as w:
w.setnchannels(1)
w.setsampwidth(2)
w.setframerate(rate)
w.writeframes(b"\x00\x00" * 240)
return buf.getvalue()
def _model(family, options = ()) -> AudioCppModel:
policy = acm.FAMILIES[family]
variant = AudioCppVariant("Q8_0", (RepoFile("m-q8_0.gguf", 1),), "m-q8_0.gguf")
return AudioCppModel(
id = f"{AUDIO_CPP_REPO}/{family}",
repo_id = AUDIO_CPP_REPO,
folder = family,
display_name = family,
family = family,
task = policy.task,
server_task = policy.default_server_task,
variant = variant,
variants = (variant,),
default_variant = "Q8_0",
options = tuple({**_BLANK_OPTION, **o} for o in options),
speaks = policy.speaks,
clone = policy.clone,
convert = policy.convert,
)
class _Server:
model_id = "studio-test"
backend = "cpu"
def __init__(
self,
model,
fail = None,
):
self.model = model
self.calls: list[tuple[str, dict]] = []
self.fail = fail
def alive(self):
return True
def stop(self):
pass
def post_json(self, path, payload, **_kwargs):
self.calls.append((path, json.loads(json.dumps(payload))))
if self.fail is not None:
raise self.fail
if path == "/v1/tasks/run":
body = {"audio": base64.b64encode(_wav(16000)).decode(), "sample_rate": 16000}
return "application/json", json.dumps(body).encode()
return "audio/wav", _wav(24000)
@pytest.fixture
def started(monkeypatch):
servers: list[_Server] = []
def start(served, path, **_kwargs):
servers.append(_Server(served))
return servers[-1]
monkeypatch.setattr(acb.AudioCppServer, "start", start)
monkeypatch.setattr(audio_cpp_files, "materialize", lambda model: "/models/m.gguf")
return servers
def _backend(model, started):
backend = acb.AudioCppBackend()
backend._model = model
backend.models = {model.id: {"is_audio": True, **acb.model_info_fields(model)}}
backend.active_model_name = model.id
backend._start_server(model)
started.clear()
return backend
def _convert(
backend,
inputs = None,
options = None,
seed = 7,
**convert,
):
wav, rate = backend.generate_audio_response(
"ignored",
workflow = "convert",
audio_inputs = {"source": SOURCE, "target": TARGET} if inputs is None else inputs,
audio_options = options,
seed = seed,
convert = convert,
)
assert wav[:4] == b"RIFF" and rate == 16000
path, body = backend._server.calls[-1]
assert path == "/v1/tasks/run"
assert body["model"] == "studio-test"
return body["request"]
# Recommended RVC defaults Unsloth Studio fills when the user leaves them unset.
_RVC_DEFAULTS = {"retrieval_blend": "0", "rms_mix_rate": "0.25", "unvoiced_protection": "0.33"}
def test_rvc_converts_to_a_builtin_voice_by_options_voice_id(started):
backend = _backend(_model("rvc", _RVC_OPTIONS), started)
request = _convert(
backend,
inputs = {"source": SOURCE},
options = {"retrieval_blend": 0.6, "voice_id": "fraise", "semitone_shift": 9},
voice = "manthos",
pitch = 3,
)
assert request == {
"audio": SOURCE,
"options": {
"retrieval_blend": "0.6",
"voice_id": "manthos",
"semitone_shift": "3",
"rms_mix_rate": "0.25",
"unvoiced_protection": "0.33",
},
}
assert _convert(backend, inputs = {"source": SOURCE}) == {
"audio": SOURCE,
"options": {"voice_id": "default", **_RVC_DEFAULTS},
}
request = _convert(backend, inputs = {"source": SOURCE}, pitch = -5, pitch_auto = True)
assert request["options"] == {"voice_id": "default", "semitone_shift": "-5", **_RVC_DEFAULTS}
assert started == []
@pytest.mark.parametrize(
"family, target, match",
[("rvc", TARGET, "built-in voices"), ("meanvc2", None, "Add the target voice.")],
)
def test_a_target_the_family_cannot_take_is_refused(family, target, match):
with pytest.raises(acc.ConvertRequestError, match = match):
acc.convert_request(_model(family), mode = "speech", source = SOURCE, target = target)
@pytest.mark.parametrize("family", ["meanvc2", "tone_color_vc"])
def test_a_reference_family_sends_audio_voice_ref_and_seed(family):
request = acc.convert_request(
_model(family), mode = "speech", source = SOURCE, target = TARGET, pitch = 4, seed = 7
)
assert request == {"audio": SOURCE, "voice_ref": TARGET, "seed": "7"}
# Recommended Seed-VC defaults Unsloth Studio fills when the user leaves them unset.
_SEED_VC_DEFAULTS = {
"length_adjust": "1",
"similarity_guidance_scale": "0.7",
"voice_anonymization": "false",
}
def test_seed_vc_speech_sends_no_pitch_and_its_route_by_model_entry(started):
model = _model("seed_vc", _SEED_VC_OPTIONS)
backend = _backend(model, started)
request = _convert(
backend,
options = {
"length_adjust": 1.25,
"similarity_guidance_scale": 1.2,
"semitone_shift": 5,
"auto_f0_adjust": True,
"f0_condition": True,
},
pitch = 5,
pitch_auto = True,
)
assert request == {
"audio": SOURCE,
"voice_ref": TARGET,
"options": {
"length_adjust": "1.25",
"similarity_guidance_scale": "1.2",
"voice_anonymization": "false",
},
"seed": "7",
}
assert started == []
request = _convert(backend, options = {"route": "v1_xlsr_hift_vc"})
assert "route" not in request and "route" not in request.get("options", {})
(server,) = started
assert server.model.server_task == "vc"
assert server.model.model_options["default_request_options"] == {"route": "v1_xlsr_hift_vc"}
assert backend.models[model.id]["audio_convert_route"] == "v1_xlsr_hift_vc"
_convert(backend, options = {"route": "v1_xlsr_hift_vc"})
assert len(started) == 1
_convert(backend)
assert len(started) == 2 and "default_request_options" not in started[-1].model.model_options
assert backend.models[model.id]["audio_convert_route"] == "v2_vc"
def test_seed_vc_singing_reloads_under_svc_and_sends_its_pitch(started):
model = _model("seed_vc", _SEED_VC_OPTIONS)
backend = _backend(model, started)
request = _convert(backend, mode = "singing", pitch = 12, pitch_auto = True)
assert request["options"] == {
"auto_f0_adjust": "true",
"semitone_shift": "12",
**_SEED_VC_DEFAULTS,
}
request = _convert(backend, mode = "singing", pitch_auto = True)
assert request["options"] == {"auto_f0_adjust": "true", **_SEED_VC_DEFAULTS}
(server,) = started
assert server.model.server_task == "svc"
assert backend.models[model.id]["audio_server_task"] == "svc"
assert backend.models[model.id]["audio_convert_route"] == "v1_svc"
request = _convert(backend, mode = "singing", pitch = -3, options = {"route": "v1_xlsr_hift_vc"})
assert request["options"] == {"semitone_shift": "-3", **_SEED_VC_DEFAULTS}
assert len(started) == 1
_convert(backend)
assert [s.model.server_task for s in started] == ["svc", "vc"]
def test_chatterbox_converts_under_vc_and_clones_back_under_clon(started):
model = _model("chatterbox")
backend = _backend(model, started)
request = _convert(
backend,
options = {"s3gen_cfg_rate": 1.0, "num_inference_steps": 6, "exaggeration": 0.9},
)
assert request == {
"audio": SOURCE,
"voice_ref": TARGET,
"options": {"s3gen_cfg_rate": "1", "num_inference_steps": "6"},
"seed": "7",
}
assert [s.model.server_task for s in started] == ["vc"]
assert backend.models[model.id]["audio_server_task"] == "vc"
_convert(backend)
assert len(started) == 1
backend.generate_audio_response(
"Hello there.", workflow = "clone", audio_inputs = {"reference": TARGET}
)
assert [s.model.server_task for s in started] == ["vc", "clon"]
assert backend.models[model.id]["audio_server_task"] == "clon"
assert started[-1].calls[-1][0] == "/v1/audio/speech"
def test_vevo2_keeps_the_source_style_with_source_audio_and_target_voice(started):
model = _model("vevo2")
backend = _backend(model, started)
assert _convert(backend, pitch_auto = True, pitch = 4) == {
"source_audio": SOURCE,
"target_voice": TARGET,
"seed": "7",
}
assert _convert(backend, pitch = 4)["options"] == {
"use_pitch_shift": "true",
"source_shift_steps": "4",
}
assert _convert(backend, pitch = 0)["options"] == {"use_pitch_shift": "false"}
assert _convert(backend, options = {"num_inference_steps": 20})["options"] == {
"num_inference_steps": "20"
}
_convert(backend, mode = "singing", pitch = -2)
assert [s.model.server_task for s in started] == ["vc", "svc"]
assert started[-1].calls[-1][1]["request"]["options"] == {
"use_pitch_shift": "true",
"source_shift_steps": "-2",
}
assert backend.models[model.id]["audio_convert_route"] is None
def test_vevo2_takes_the_target_style_through_the_source_transcript():
model = _model("vevo2")
request = acc.convert_request(
model,
mode = "speech",
source = SOURCE,
target = TARGET,
style = "target",
source_text = f" {TRANSCRIPT} ",
pitch = 5,
seed = 7,
)
assert request == {
"source_audio": SOURCE,
"target_voice": TARGET,
"route": "style_converted_vc",
"target_text": TRANSCRIPT,
"style_ref": TARGET,
"seed": "7",
}
with pytest.raises(acc.ConvertRequestError, match = "Type what's said in the recording"):
acc.convert_request(model, mode = "speech", source = SOURCE, target = TARGET, style = "target")
singing = acc.convert_request(
model, mode = "singing", source = SOURCE, target = TARGET, style = "target"
)
assert "route" not in singing and "target_text" not in singing
def test_a_mode_the_model_lacks_is_refused(started):
backend = _backend(_model("meanvc2"), started)
with pytest.raises(RuntimeError, match = "does not convert singing"):
_convert(backend, mode = "singing")
with pytest.raises(acc.ConvertRequestError):
acc.served_model(_model("kokoro_tts"), "speech", {})
def test_a_runtime_refusal_is_an_audio_runtime_error(started):
backend = _backend(_model("meanvc2"), started)
backend._server.fail = srv.AudioCppRequestError(500, "MeanVC2 requires --voice-ref target")
with pytest.raises(AudioRuntimeError, match = "MeanVC2 requires --voice-ref target") as info:
_convert(backend)
assert info.value.status == 500
def test_status_fields_describe_convert_per_family():
rvc = acb.model_info_fields(_model("rvc", _RVC_OPTIONS))
assert rvc["audio_workflows"] == ["convert"]
assert rvc["audio_convert"] == {
"modes": ["speech"],
"target": "builtin",
"builtin_voices": [
{"id": "default", "label": "Default"},
{"id": "manthos", "label": "Manthos"},
{"id": "chocola", "label": "Chocola"},
{"id": "fraise", "label": "Fraise"},
],
"pitch": {"speech": {"auto": False, "shift_with_auto": False}},
"style": False,
"route_reloads": False,
"source_max_seconds": 300,
}
assert rvc["audio_workflow_tasks"] == {"convert": "vc"}
assert (rvc["audio_server_task"], rvc["audio_convert_route"]) == ("vc", None)
assert [o["name"] for o in rvc["audio_options_by_workflow"]["convert"]] == [
"retrieval_blend",
"rms_mix_rate",
"unvoiced_protection",
]
assert rvc["audio_convert_rules"] == {"source_rate": 16000, "target_rate": None}
seed_vc = acb.model_info_fields(_model("seed_vc", _SEED_VC_OPTIONS))
assert seed_vc["audio_convert"]["modes"] == ["speech", "singing"]
assert seed_vc["audio_convert"]["pitch"] == {"singing": {"auto": True, "shift_with_auto": True}}
assert seed_vc["audio_convert"]["route_reloads"] is True
assert seed_vc["audio_workflow_tasks"] == {"convert": "vc", "convert:singing": "svc"}
assert seed_vc["audio_convert_route"] == "v2_vc"
assert seed_vc["audio_convert_rules"] == {"source_rate": 44100, "target_rate": 44100}
vevo2 = acb.model_info_fields(_model("vevo2"))
assert vevo2["audio_workflows"] == ["clone", "convert"]
assert vevo2["audio_convert"]["pitch"] == {
"speech": {"auto": True, "shift_with_auto": False},
"singing": {"auto": True, "shift_with_auto": False},
}
assert vevo2["audio_convert"]["style"] is True
assert vevo2["audio_workflow_tasks"] == {
"clone": "tts",
"convert": "vc",
"convert:singing": "svc",
}
assert vevo2["audio_server_task"] == "tts"
chatterbox = acb.model_info_fields(_model("chatterbox"))
assert chatterbox["audio_workflow_tasks"] == {"clone": "clon", "convert": "vc"}
assert chatterbox["audio_convert"]["pitch"] == {}
assert chatterbox["audio_convert_rules"] == {"source_rate": 16000, "target_rate": 24000}
assert acb.model_info_fields(_model("meanvc2"))["audio_convert"]["pitch"] == {}
tone_color = acb.model_info_fields(_model("tone_color_vc"))
assert tone_color["audio_workflow_tasks"] == {"convert": "vc"}
assert tone_color["audio_convert_rules"] == {"source_rate": 22050, "target_rate": 22050}
kokoro = acb.model_info_fields(_model("kokoro_tts"))
assert kokoro["audio_convert"] is None and kokoro["audio_options_by_workflow"] is None
assert kokoro["audio_workflow_tasks"] == {"speak": "tts"}
def test_the_seed_vc_server_entry_carries_its_route(monkeypatch, tmp_path):
seen: dict = {}
binary = tmp_path / srv.BINARY_NAME
binary.write_bytes(b"")
def spy(command, *_args, **_kwargs):
with open(command[command.index("--config") + 1], encoding = "utf-8") as f:
seen.update(json.load(f))
raise OSError("not launched in tests")
monkeypatch.setattr(srv, "ensure_binary", lambda: str(binary))
monkeypatch.setattr(srv, "model_runtime_problem", lambda model, binary = None: None)
monkeypatch.setattr(srv, "select_backend", lambda binary, force_cpu: "cpu")
monkeypatch.setattr(srv.subprocess, "Popen", spy)
served, rest = acc.served_model(
_model("seed_vc", _SEED_VC_OPTIONS), "speech", {"route": "v1_whisper_bigvgan_vc", "x": 1}
)
assert rest == {"x": 1}
with pytest.raises(srv.AudioCppUnavailableError):
srv.AudioCppServer.start(served, "/models/m.gguf")
(entry,) = seen["models"]
assert entry["task"] == "vc"
assert entry["default_request_options"] == {"route": "v1_whisper_bigvgan_vc"}
def test_a_five_minute_source_transcript_fits_the_request():
from models.inference import AudioRunInputs
text = "word " * 1100
inputs = AudioRunInputs(source = {"input_id": "a" * 32}, source_text = text)
assert inputs.source_text == text