Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: load the speech model an audio API request asks for (#10217)

Nilay2e6db0c1289d
diff --git a/studio/backend/core/inference/audio_codecs.py b/studio/backend/core/inference/audio_codecs.py
index 89ba21284f8..bf116b911ea 100644
--- a/studio/backend/core/inference/audio_codecs.py
+++ b/studio/backend/core/inference/audio_codecs.py
@@ -6,11 +6,16 @@ Audio codec loading and decoding for TTS inference.
Supports: SNAC (Orpheus), CSM (Sesame), BiCodec (Spark), DAC (OuteTTS)
"""
+from __future__ import annotations
+
import io
+import json
+import os
import re
import wave
import structlog
from loggers import get_logger
+from pathlib import Path
from typing import Optional, Tuple
import numpy as np
@@ -26,6 +31,86 @@ from utils.third_party_source import (
)
logger = get_logger(__name__)
+_SPARK_TTS_REPO = "unsloth/Spark-TTS-0.5B"
+_MAX_SPARK_EXPORT_METADATA_BYTES = 1_000_000
+
+
+def _bicodec_assets_complete(repo_path: Path) -> bool:
+ try:
+ config = repo_path / "BiCodec" / "config.yaml"
+ weights = repo_path / "BiCodec" / "model.safetensors"
+ return (
+ config.is_file()
+ and config.stat().st_size > 0
+ and weights.is_file()
+ and weights.stat().st_size > 0
+ )
+ except OSError:
+ return False
+
+
+def resolve_bicodec_repo_path(
+ model_repo_path: Optional[str] = None,
+ *,
+ hf_token: Optional[str] | bool = None,
+ local_files_only: Optional[bool] = None,
+ cache_dir: Optional[str] = None,
+) -> str:
+ from huggingface_hub import snapshot_download
+ from utils.hf_cache_settings import active_hf_hub_cache
+ from utils.utils import canonical_model_repo_id, hf_env_offline
+
+ source = str(model_repo_path or _SPARK_TTS_REPO).strip()
+ local = Path(source).expanduser()
+ if local.is_dir():
+ root = local.parent if local.name.lower() == "llm" else local
+ if _bicodec_assets_complete(root):
+ ensure_spark_tts_source(root)
+ return os.path.abspath(root)
+
+ metadata = local / "export_metadata.json"
+ base_model = None
+ try:
+ if metadata.is_file() and metadata.stat().st_size <= _MAX_SPARK_EXPORT_METADATA_BYTES:
+ value = json.loads(metadata.read_text(encoding = "utf-8-sig"))
+ candidate = value.get("base_model") if isinstance(value, dict) else None
+ if isinstance(candidate, str) and candidate.strip():
+ base_model = candidate.strip()
+ except (OSError, UnicodeError, TypeError, ValueError):
+ pass
+ if base_model is None:
+ raise RuntimeError("The local Spark-TTS model has no complete BiCodec assets")
+ source = base_model
+ local = Path(source).expanduser()
+ if local.is_dir():
+ root = local.parent if local.name.lower() == "llm" else local
+ if not _bicodec_assets_complete(root):
+ raise RuntimeError("The recorded Spark-TTS base has incomplete BiCodec assets")
+ ensure_spark_tts_source(root)
+ return os.path.abspath(root)
+
+ from utils.security import load_scan_target
+
+ repo_id, _load_subdirs = load_scan_target(canonical_model_repo_id(source), ())
+ repo_id = repo_id or source
+ root = Path(
+ snapshot_download(
+ repo_id,
+ token = (
+ False
+ if hf_token is False
+ else hf_token.strip()
+ if hf_token and hf_token.strip()
+ else None
+ ),
+ cache_dir = cache_dir or active_hf_hub_cache(),
+ local_files_only = hf_env_offline() if local_files_only is None else local_files_only,
+ )
+ )
+ if not _bicodec_assets_complete(root):
+ raise RuntimeError("The staged Spark-TTS repository has incomplete BiCodec assets")
+ ensure_spark_tts_source(root)
+ return os.path.abspath(root)
def _numpy_to_wav_bytes(waveform: np.ndarray, sample_rate: int) -> bytes:
@@ -68,11 +153,11 @@ class AudioCodecManager:
) -> None:
"""Load the appropriate codec for the given audio type."""
if audio_type == "snac":
- self._load_snac(device)
+ self._load_snac(device, model_repo_path)
elif audio_type == "bicodec":
self._load_bicodec(device, model_repo_path)
elif audio_type == "dac":
- self._load_dac(device)
+ self._load_dac(device, model_repo_path)
elif audio_type == "csm":
pass # CSM decoding is built into the model (output_audio=True)
else:
@@ -80,7 +165,11 @@ class AudioCodecManager:
# ── Lazy loaders ─────────────────────────────────────────────
- def _load_snac(self, device: str) -> None:
+ def _load_snac(
+ self,
+ device: str,
+ model_repo_path: Optional[str] = None,
+ ) -> None:
if self._snac_model is not None:
return
from snac import SNAC
@@ -88,7 +177,10 @@ class AudioCodecManager:
# Route weights to the selected cache; this can run in the main process.
self._snac_model = (
- SNAC.from_pretrained("hubertsiuzdak/snac_24khz", cache_dir = active_hf_hub_cache())
+ SNAC.from_pretrained(
+ model_repo_path or "hubertsiuzdak/snac_24khz",
+ cache_dir = active_hf_hub_cache(),
+ )
.to(device)
.eval()
)
@@ -116,7 +208,11 @@ class AudioCodecManager:
self._codec_devices["bicodec"] = device
logger.info(f"Loaded BiCodec tokenizer from {tokenizer_path}")
- def _load_dac(self, device: str) -> None:
+ def _load_dac(
+ self,
+ device: str,
+ audio_codec_path: Optional[str] = None,
+ ) -> None:
if self._dac_audio_codec is not None:
return
outetts_code_dir = ensure_outetts_source()
@@ -129,12 +225,12 @@ class AudioCodecManager:
"outetts.models.config",
outetts_code_dir,
).ModelConfig
- audio_codec_path = ensure_dac_speech_weights()
+ resolved_audio_codec_path = audio_codec_path or ensure_dac_speech_weights()
dummy_config = OuteTTSModelConfig(
tokenizer_path = None,
device = device,
- audio_codec_path = str(audio_codec_path),
+ audio_codec_path = str(resolved_audio_codec_path),
)
processor = AudioProcessor(config = dummy_config)
self._dac_audio_codec = processor.audio_codec
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index b02f5dfd9fb..f8d0a0d9850 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -633,6 +633,10 @@ def neutralize_tts_prompt_text(text: str, audio_type = None) -> str:
return _spaced_out(_TTS_MARKUP_BY_CODEC.get(audio_type, _TTS_MARKUP_DEFAULT), text)
+def build_dac_tts_prompt(text: str) -> str:
+ return f"<|im_start|>\n<|text_start|>{text}<|text_end|>\n<|audio_start|>\n"
+
+
def _neutralize_leaves(
value,
rewrite,
diff --git a/studio/backend/core/inference/diffusion_compat.py b/studio/backend/core/inference/diffusion_compat.py
index f373a4d59b3..9fa63c33748 100644
--- a/studio/backend/core/inference/diffusion_compat.py
+++ b/studio/backend/core/inference/diffusion_compat.py
@@ -241,8 +241,16 @@ def _interrupt_read(response: Any) -> None:
pass
-def _read_gguf_header(repo_id: str, gguf_filename: str, hf_token: Optional[str]) -> bytes:
- """The first ``_GGUF_HEADER_BYTES`` of a Hub-hosted GGUF, or b"" when they cannot be read.
+def _read_gguf_header(
+ repo_id: str,
+ gguf_filename: str,
+ hf_token: Optional[str],
+ *,
+ revision: Optional[str] = None,
+ max_bytes: Optional[int] = None,
+ timeout_seconds: Optional[float] = None,
+) -> bytes:
+ """A bounded prefix of a Hub-hosted GGUF, or b"" when it cannot be read.
One wall-clock bound over the WHOLE operation, request included. requests' timeout is an
inactivity timeout: a peer (or an intermediary) that trickles response HEADERS resets it on
@@ -258,6 +266,8 @@ def _read_gguf_header(repo_id: str, gguf_filename: str, hf_token: Optional[str])
from huggingface_hub.utils import build_hf_headers, get_session
except Exception: # noqa: BLE001 — an unexpected hub layout leaves today's behaviour
return b""
+ max_bytes = _GGUF_HEADER_BYTES if max_bytes is None else max_bytes
+ timeout_seconds = _HEADER_TIMEOUT_SECONDS if timeout_seconds is None else timeout_seconds
buffer = bytearray()
# Published by the worker as soon as it has something interruptible; read by this thread on
# timeout. A one-element list rather than a nonlocal, so the worker's assignment is visible.
@@ -266,9 +276,9 @@ def _read_gguf_header(repo_id: str, gguf_filename: str, hf_token: Optional[str])
def _fetch() -> None:
try:
headers = dict(build_hf_headers(token = hf_token))
- headers["Range"] = f"bytes=0-{_GGUF_HEADER_BYTES - 1}"
+ headers["Range"] = f"bytes=0-{max_bytes - 1}"
with _ranged_stream(
- get_session(), hf_hub_url(repo_id, gguf_filename), headers
+ get_session(), hf_hub_url(repo_id, gguf_filename, revision = revision), headers
) as response:
holder[0] = response
# 206 or nothing. A server (or a proxy) that ignored the Range header answers 200
@@ -276,12 +286,12 @@ def _read_gguf_header(repo_id: str, gguf_filename: str, hf_token: Optional[str])
# download this preflight exists to prevent.
if response.status_code != 206:
return
- deadline = time.monotonic() + _HEADER_TIMEOUT_SECONDS
+ deadline = time.monotonic() + timeout_seconds
for chunk in _iter_body(response, 65536):
# extend, not `+=`: augmented assignment to a closed-over name would rebind
# it as a local of _fetch and lose every byte.
buffer.extend(chunk)
- if len(buffer) >= _GGUF_HEADER_BYTES or time.monotonic() > deadline:
+ if len(buffer) >= max_bytes or time.monotonic() > deadline:
break
# Keep what arrived rather than discarding it: the deadline firing on a merely SLOW link
# still leaves the tensor table (the first ~15 KiB) in hand, and the parser is
@@ -298,19 +308,19 @@ def _read_gguf_header(repo_id: str, gguf_filename: str, hf_token: Optional[str])
# return -- on urllib3 >= 2.3, where HTTPResponse.shutdown exists. requirements/studio.txt
# floors it, but an install predating that floor keeps whatever it resolved, so the bound
# here cannot depend on the version underneath us: the worker is abandonable either way.
- watchdog = threading.Timer(_HEADER_TIMEOUT_SECONDS, lambda: _interrupt_read(holder[0]))
+ watchdog = threading.Timer(timeout_seconds, lambda: _interrupt_read(holder[0]))
watchdog.daemon = True
watchdog.start()
worker = threading.Thread(target = _fetch, name = "gguf-header-read", daemon = True)
worker.start()
- worker.join(_HEADER_TIMEOUT_SECONDS)
+ worker.join(timeout_seconds)
if worker.is_alive():
_interrupt_read(holder[0])
worker.join(_ABANDON_GRACE_SECONDS)
watchdog.cancel()
# bytes() snapshots under the GIL, so an abandoned worker still appending cannot tear the
# copy; it can only lose a chunk that arrived too late to matter.
- return bytes(buffer[:_GGUF_HEADER_BYTES])
+ return bytes(buffer[:max_bytes])
def flux2_inner_dim_for_pick(
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 24b87891dce..b9094102bcc 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -3,6 +3,8 @@
"""Core inference backend."""
+from __future__ import annotations
+
from unsloth import FastLanguageModel, FastVisionModel
from unsloth.chat_templates import get_chat_template
from transformers import TextIteratorStreamer, TextStreamer
@@ -34,6 +36,7 @@ from core.inference.chat_eos import (
resolve_chat_turn_end_eos_ids_using,
)
from core.inference.chat_template_helpers import (
+ build_dac_tts_prompt,
make_reasoning_normalizer,
detect_reasoning_channel_markers,
detect_think_prefill,
@@ -56,6 +59,12 @@ from loggers import get_logger
logger = get_logger(__name__)
+def _hf_token_for_loader(hf_token: Optional[str] | bool) -> Optional[str] | bool:
+ if hf_token is False:
+ return False
+ return hf_token.strip() if isinstance(hf_token, str) and hf_token.strip() else None
+
+
class HarmonyTextStreamer:
"""Streaming text decoder for the gpt-oss harmony channel protocol.
@@ -334,9 +343,10 @@ class InferenceBackend:
max_seq_length: int = 2048,
dtype = None,
load_in_4bit: bool = True,
- hf_token: Optional[str] = None,
+ hf_token: Optional[str] | bool = None,
trust_remote_code: bool = False,
gpu_ids: Optional[list[int]] = None,
+ audio_codec_path: Optional[str] = None,
) -> bool:
"""Load any model: base, LoRA adapter, text, or vision."""
# Keep the token so the native-template fallback can fetch a
@@ -404,7 +414,7 @@ class InferenceBackend:
auto_model = CsmForConditionalGeneration,
load_in_4bit = False,
device_map = device_map,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastModel.for_inference(model)
@@ -413,19 +423,16 @@ class InferenceBackend:
self.models[model_name]["processor"] = processor
elif audio_type == "bicodec":
import os
+ from core.inference.audio_codecs import resolve_bicodec_repo_path
from unsloth import FastModel
if config.is_lora and config.base_model:
# LoRA adapter: base_model is .../Spark-TTS-0.5B/LLM;
# BiCodec weights live in the parent dir.
base_path = config.base_model
- if os.path.isdir(base_path):
- abs_repo_path = os.path.abspath(os.path.dirname(base_path))
- else:
- # base_model is an HF ID — download it.
- from huggingface_hub import snapshot_download
- repo_path = snapshot_download(base_path)
- abs_repo_path = os.path.abspath(repo_path)
+ abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
+ base_path, hf_token = hf_token
+ )
logger.info(
f"Spark-TTS LoRA: loading adapter from {config.path}, BiCodec from {abs_repo_path}"
@@ -437,7 +444,7 @@ class InferenceBackend:
attn_implementation = "sdpa",
load_in_4bit = False,
device_map = device_map,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
elif config.is_local and os.path.isdir(config.path):
@@ -449,57 +456,21 @@ class InferenceBackend:
llm_path = os.path.join(config.path, "LLM")
if not os.path.isdir(llm_path):
llm_path = config.path
- base_repo = None
- try:
- meta_path = Path(config.path) / "export_metadata.json"
- if meta_path.exists():
- base_repo = json.loads(
- meta_path.read_text(encoding = "utf-8-sig")
- ).get("base_model")
- except Exception:
- base_repo = None
- if base_repo and os.path.isdir(base_repo):
- # A base recorded as .../Spark-TTS-0.5B/LLM keeps BiCodec in its parent.
- abs_repo_path = os.path.abspath(
- os.path.dirname(base_repo)
- if os.path.basename(base_repo.rstrip("/\\")) == "LLM"
- else base_repo
- )
- elif base_repo:
- from huggingface_hub import snapshot_download
-
- # Registry alias ("Spark-TTS-0.5B/LLM") names a load
- # subdirectory, not a repo, so snapshot_download rejects it.
- # Same resolver the capability probe and the trainer preflight
- # use, rather than a second copy of the mapping.
- from utils.security import load_scan_target
- from utils.utils import canonical_model_repo_id
-
- hf_repo, _load_subdirs = load_scan_target(
- canonical_model_repo_id(base_repo), ()
- )
- hf_repo = hf_repo or base_repo
- # Same token as the load below: a private or gated base would
- # otherwise 401 here while resolving the BiCodec assets.
- abs_repo_path = os.path.abspath(
- snapshot_download(
- hf_repo,
- token = hf_token if hf_token and hf_token.strip() else None,
- )
- )
- else:
- abs_repo_path = os.path.abspath(config.path)
+ abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
+ config.path, hf_token = hf_token
+ )
logger.info(
f"Spark-TTS merged export: LLM from {llm_path}, BiCodec from {abs_repo_path}"
)
else:
# Base model: download full HF repo, load from /LLM subfolder
- from huggingface_hub import snapshot_download
-
- repo_path = snapshot_download(config.path)
- abs_repo_path = os.path.abspath(repo_path)
+ abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
+ config.path, hf_token = hf_token
+ )
llm_path = os.path.join(abs_repo_path, "LLM")
- logger.info(f"Spark-TTS: repo at {repo_path}, loading LLM from {llm_path}")
+ logger.info(
+ f"Spark-TTS: repo at {abs_repo_path}, loading LLM from {llm_path}"
+ )
if not (config.is_lora and config.base_model):
# Shared by the merged-export and repo-root branches above: both resolve
@@ -511,7 +482,7 @@ class InferenceBackend:
attn_implementation = "sdpa",
load_in_4bit = False,
device_map = device_map,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
@@ -528,7 +499,7 @@ class InferenceBackend:
max_seq_length = max_seq_length,
load_in_4bit = False,
device_map = device_map,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastModel.for_inference(model)
@@ -546,7 +517,7 @@ class InferenceBackend:
whisper_task = "transcribe",
load_in_4bit = False,
device_map = device_map,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastModel.for_inference(model)
@@ -574,7 +545,7 @@ class InferenceBackend:
max_seq_length = max_seq_length,
load_in_4bit = False,
device_map = device_map,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastLanguageModel.for_inference(model)
@@ -584,7 +555,9 @@ class InferenceBackend:
# Load external codec for TTS audio types
# (Whisper is ASR, audio_vlm is audio input — neither needs one)
if audio_type not in ("whisper", "audio_vlm"):
- model_repo_path = self.models[model_name].get("model_repo_path")
+ model_repo_path = audio_codec_path or self.models[model_name].get(
+ "model_repo_path"
+ )
self._audio_codec_manager.load_codec(
audio_type, self.device, model_repo_path = model_repo_path
)
@@ -616,7 +589,7 @@ class InferenceBackend:
dtype = dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
@@ -648,7 +621,7 @@ class InferenceBackend:
processor = AutoProcessor.from_pretrained(
processor_source,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
logger.info(f"Loaded {type(processor).__name__} from {processor_source}")
@@ -665,7 +638,7 @@ class InferenceBackend:
dtype = dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
- token = hf_token if hf_token and hf_token.strip() else None,
+ token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
@@ -2364,11 +2337,7 @@ class InferenceBackend:
# (same as the OuteTTS notebook) to avoid degenerate repetition.
self._patch_repetition_penalty_processor()
- prompt = (
- "<|im_start|>\n<|text_start|>"
- + text
- + "<|text_end|>\n<|audio_start|><|global_features_start|>\n"
- )
+ prompt = build_dac_tts_prompt(text)
with torch.inference_mode():
# Derive the autocast device from the loaded model, not from the
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index ecc6a6d3e67..af1c156176c 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -50,6 +50,7 @@ from typing import (
import httpx
+from core.inference.chat_template_helpers import build_dac_tts_prompt
from core.inference.context_window import (
_COMPACTION_HEADROOM_RATIO,
clamp_compaction_headroom_ratio,
@@ -518,6 +519,7 @@ class GgufLoadIntent:
hf_repo: Optional[str] = None
hf_variant: Optional[str] = None
hf_token: Optional[str] = None
+ audio_codec_path: Optional[str] = None
is_vision: bool = False
# Load a vision GGUF as text-only: no projector on the GPU and none on the CPU
# either. The projector's VRAM is left for the model, and image input is off
@@ -18554,7 +18556,12 @@ class LlamaCppBackend:
except Exception as exc:
logger.debug("Fast-path audio probe failed: %s", exc)
detected = None
- if not self._apply_detected_audio(detected):
+ applied_audio = (
+ self._apply_detected_audio(detected)
+ if intent.audio_codec_path is None
+ else self._apply_detected_audio(detected, intent.audio_codec_path)
+ )
+ if not applied_audio:
return False
if not self._healthy:
return False
@@ -25334,7 +25341,12 @@ class LlamaCppBackend:
except Exception as exc:
logger.debug("Audio probe failed: %s", exc)
detected = None
- if not self._apply_detected_audio(detected):
+ applied_audio = (
+ self._apply_detected_audio(detected)
+ if intent.audio_codec_path is None
+ else self._apply_detected_audio(detected, intent.audio_codec_path)
+ )
+ if not applied_audio:
return False
if not self._healthy:
@@ -33101,7 +33113,11 @@ class LlamaCppBackend:
logger.debug(f"Audio type detection failed: {e}")
return None
- def _apply_detected_audio(self, detected: Optional[str]) -> bool:
+ def _apply_detected_audio(
+ self,
+ detected: Optional[str],
+ audio_codec_path: Optional[str] = None,
+ ) -> bool:
"""Apply a probed audio codec under self._lock. Returns True to continue
the load (codec inited OK, or nothing to init), False to abort (server
unhealthy or codec init failed). Shared by the fast-path retry and the
@@ -33111,7 +33127,7 @@ class LlamaCppBackend:
if not self._healthy:
return False
try:
- self.init_audio_codec(detected)
+ self.init_audio_codec(detected, audio_codec_path)
self._is_audio = True
self._audio_type = detected
except Exception as exc:
@@ -33187,7 +33203,7 @@ class LlamaCppBackend:
False,
),
"dac": (
- "<|im_start|>\n<|text_start|>{text}<|text_end|>\n<|audio_start|><|global_features_start|>\n",
+ build_dac_tts_prompt("{text}"),
["<|im_end|>", "<|audio_end|>"],
False,
),
@@ -33206,7 +33222,11 @@ class LlamaCppBackend:
if torch.cuda.is_available():
torch.cuda.empty_cache()
- def init_audio_codec(self, audio_type: str) -> None:
+ def init_audio_codec(
+ self,
+ audio_type: str,
+ audio_codec_path: Optional[str] = None,
+ ) -> None:
"""Load the audio codec at model load time (mirrors the non-GGUF path)."""
import torch
from core.inference.audio_codecs import AudioCodecManager
@@ -33218,15 +33238,13 @@ class LlamaCppBackend:
# load is classified as not holding, which is what lets the route skip
# arbitration and survive training.
device = "cuda" if torch.cuda.is_available() and not self.holds_no_vram else "cpu"
- model_repo_path = None
+ model_repo_path = audio_codec_path
# BiCodec needs a repo with BiCodec/ weights -- download canonical SparkTTS
- if audio_type == "bicodec":
- from huggingface_hub import snapshot_download
- import os
-
- repo_path = snapshot_download("unsloth/Spark-TTS-0.5B")
- model_repo_path = os.path.abspath(repo_path)
+ if audio_type == "bicodec" and model_repo_path is None:
+ from core.inference.audio_codecs import resolve_bicodec_repo_path
+ from utils.utils import hf_env_offline
+ model_repo_path = resolve_bicodec_repo_path(local_files_only = hf_env_offline())
LlamaCppBackend._codec_mgr.load_codec(audio_type, device, model_repo_path = model_repo_path)
logger.info(f"Loaded audio codec for GGUF TTS: {audio_type}")
diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py
index 42a33f69ec8..d871981e971 100644
--- a/studio/backend/core/inference/local_model_resolver.py
+++ b/studio/backend/core/inference/local_model_resolver.py
@@ -88,21 +88,31 @@ def _advertised_loader_id(info) -> Optional[str]:
return public_model_id(raw_id) or raw_id
-def _resolve_load_dir(p):
- """The concrete dir holding the GGUFs. For an HF cache repo (``models--*``
- with ``snapshots/``) this is the latest snapshot dir, so /load takes the
- local branch instead of the download-capable repo-id branch."""
+def _resolve_load_dir(p, loader_id: Optional[str] = None):
from pathlib import Path
+ load_dir = p
try:
if (p / "snapshots").is_dir():
from routes.models import _resolve_hf_cache_realpath
real = _resolve_hf_cache_realpath(p)
if real:
- return Path(real)
+ load_dir = Path(real)
except Exception:
pass
- return p
+ try:
+ from utils.audio_tokens import detect_local_tts_audio_type
+ from utils.models.model_config import load_model_defaults
+
+ llm_dir = load_dir / "LLM"
+ if llm_dir.is_dir() and (
+ (loader_id and (load_model_defaults(loader_id) or {}).get("audio_type") == "bicodec")
+ or detect_local_tts_audio_type(llm_dir) == "bicodec"
+ ):
+ return llm_dir
+ except Exception:
+ pass
+ return load_dir
def _legacy_variant_aliases(variants) -> tuple[tuple[str, str], ...]:
@@ -346,6 +356,52 @@ def _config_is_servable_here(load_dir, config: dict) -> bool:
return _is_generative_chat_config(config)
+def _host_can_serve_minimax_music3() -> bool:
+ import sys
+ from importlib.util import find_spec
+ from importlib.metadata import version
+ from packaging.version import Version
+
+ try:
+ from core.inference.audio_device import audio_device_forces_cpu
+ from utils.hardware import hardware as hw
+ return (
+ sys.version_info >= (3, 10)
+ and hw.DEVICE == hw.DeviceType.CUDA
+ and not hw.IS_ROCM
+ and not audio_device_forces_cpu(None)
+ and find_spec("diffusers") is not None
+ and Version(version("diffusers")) >= Version("0.40.0")
+ )
+ except Exception:
+ return False
+
+
+def _native_audio_pipeline_is_servable_here(load_dir) -> bool:
+ from core.inference.native_audio import (
+ _MINIMAX_DOWNLOAD_COMPONENTS,
+ minimax_music3_local_components_complete,
+ native_audio_type_from_local_path,
+ )
+ from utils.security.remote_code_scan import REMOTE_CODE_CONFIG_FILES
+
+ if (
+ native_audio_type_from_local_path(str(load_dir)) != "minimax_music3"
+ or not _host_can_serve_minimax_music3()
+ or not minimax_music3_local_components_complete(load_dir)
+ ):
+ return False
+ for directory in (
+ load_dir,
+ *(load_dir / component for component in _MINIMAX_DOWNLOAD_COMPONENTS),
+ ):
+ for name in REMOTE_CODE_CONFIG_FILES:
+ config = _read_json(directory / name)
+ if isinstance(config, dict) and config.get("auto_map"):
+ return False
+ return not any((load_dir / name).is_file() for name in (*_ADAPTER_MARKERS, "modules.json"))
+
+
def _local_weights_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
"""Build an entry for a local non-GGUF checkpoint (safetensors, MLX) the
inference orchestrator can serve, else None.
@@ -366,7 +422,9 @@ def _local_weights_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
p = Path(path)
if not p.is_dir():
return None
- load_dir = _resolve_load_dir(p)
+ load_dir = _resolve_load_dir(p, loader_id)
+ if _native_audio_pipeline_is_servable_here(load_dir):
+ return _LocalGgufEntry(loader_id, str(load_dir), (), is_gguf = False)
if not _weights_are_servable(load_dir):
return None
config = _read_json(load_dir / "config.json")
diff --git a/studio/backend/core/inference/native_audio.py b/studio/backend/core/inference/native_audio.py
index 2c7980459c3..e955099ec97 100644
--- a/studio/backend/core/inference/native_audio.py
+++ b/studio/backend/core/inference/native_audio.py
@@ -33,7 +33,7 @@ import sys
import tempfile
import threading
from contextlib import contextmanager
-from pathlib import Path
+from pathlib import Path, PurePosixPath, PureWindowsPath
from typing import Any, Optional, Tuple
logger = logging.getLogger(__name__)
@@ -92,20 +92,141 @@ def _minimax_lyrics_for_pipeline(lyrics: str) -> str:
return "\n".join(normalized)
-_MINIMAX_DOWNLOAD_COMPONENTS = frozenset(
- (
- "condition_encoder",
- "language_model",
- "rvq_depth_decoder",
- "scheduler",
- "tokenizer",
- "transformer",
- "vocoder",
- )
-)
+_MINIMAX_COMPONENT_SPECS = {
+ "condition_encoder": ("diffusers", "MiniMaxMusic3ConditionEncoder", "diffusion_pytorch_model"),
+ "language_model": ("transformers", "Qwen3ForCausalLM", "model"),
+ "rvq_depth_decoder": (
+ "diffusers",
+ "MiniMaxMusic3RVQDepthDecoder",
+ "diffusion_pytorch_model",
+ ),
+ "scheduler": ("diffusers", "FlowMatchEulerDiscreteScheduler", None),
+ "tokenizer": ("transformers", "Qwen2Tokenizer", None),
+ "transformer": (
+ "diffusers",
+ "MiniMaxMusic3Transformer1DModel",
+ "diffusion_pytorch_model",
+ ),
+ "vocoder": ("diffusers", "MiniMaxMusic3Vocoder", "diffusion_pytorch_model"),
+}
+_MINIMAX_DOWNLOAD_COMPONENTS = frozenset(_MINIMAX_COMPONENT_SPECS)
_MOSS_CONFIG_COMPAT_LOCK = threading.Lock()
_MOSS_NANO_SAVE_LOCK = threading.Lock()
_MAX_AUDIO_METADATA_BYTES = 1_000_000
+_MAX_MINIMAX_TOKENIZER_BYTES = 32 * 1024 * 1024
+
+
+def _minimax_component_has_weights(directory: Path, weight_stem: str) -> bool:
+ try:
+ index_path = directory / f"{weight_stem}.safetensors.index.json"
+ indexes = (index_path,) if index_path.is_file() else ()
+ except OSError:
+ return False
+ for index_path in indexes:
+ try:
+ if (
+ index_path.stat().st_size <= 0
+ or index_path.stat().st_size > _MAX_AUDIO_METADATA_BYTES
+ ):
+ continue
+ index = json.loads(index_path.read_text(encoding = "utf-8-sig"))
+ weight_map = index.get("weight_map") if isinstance(index, dict) else None
+ if not isinstance(weight_map, dict) or not weight_map:
+ continue
+ shards = set(weight_map.values())
+ if not all(isinstance(shard, str) and shard for shard in shards):
+ continue
+ complete = True
+ for shard in shards:
+ posix = PurePosixPath(shard.replace("\\", "/"))
+ windows = PureWindowsPath(shard)
+ if (
+ posix.is_absolute()
+ or ".." in posix.parts
+ or windows.is_absolute()
+ or windows.drive
+ or posix.suffix.lower() != ".safetensors"
+ ):
+ complete = False
+ break
+ target = directory.joinpath(*posix.parts)
+ if not target.is_file() or target.stat().st_size <= 0:
+ complete = False
+ break
+ if complete:
+ return True
+ except (OSError, TypeError, ValueError, RecursionError):
+ continue
+ if indexes:
+ return False
+ try:
+ weights = directory / f"{weight_stem}.safetensors"
+ return weights.is_file() and weights.stat().st_size > 0
+ except OSError:
+ return False
+
+
+def higgs_tts2_codec_local_complete(codec_path: str) -> bool:
+ try:
+ root = Path(codec_path).expanduser()
+ if root.is_file():
+ root = root.parent
+ config = _read_local_audio_metadata(root, "config.json", reject_oversized = True)
+ return str(
+ config.get("model_type") or ""
+ ).lower() == "higgs_audio_v2_tokenizer" and _minimax_component_has_weights(root, "model")
+ except (OSError, TypeError, ValueError, json.JSONDecodeError):
+ return False
+
+
+def minimax_music3_local_components_complete(model_path) -> bool:
+ try:
+ root = Path(model_path).expanduser()
+ if root.is_file():
+ root = root.parent
+ index = _read_local_audio_metadata(root, "modular_model_index.json")
+ for component in _MINIMAX_DOWNLOAD_COMPONENTS:
+ entry = index.get(component)
+ expected_library, expected_class, weight_stem = _MINIMAX_COMPONENT_SPECS[component]
+ metadata = (
+ next(
+ (part for part in entry if isinstance(part, dict)),
+ None,
+ )
+ if isinstance(entry, list)
+ else None
+ )
+ if (
+ not isinstance(entry, list)
+ or len(entry) < 3
+ or entry[:2] != [expected_library, expected_class]
+ or not isinstance(metadata, dict)
+ or metadata.get("subfolder") != component
+ ):
+ return False
+ directory = root / component
+ if weight_stem is not None:
+ if not _read_local_audio_metadata(directory, "config.json", reject_oversized = True):
+ return False
+ if not _minimax_component_has_weights(directory, weight_stem):
+ return False
+ elif component == "scheduler":
+ if not _read_local_audio_metadata(
+ directory, "scheduler_config.json", reject_oversized = True
+ ):
+ return False
+ elif not _read_local_audio_metadata(
+ directory, "tokenizer_config.json", reject_oversized = True
+ ) or not _read_local_audio_metadata(
+ directory,
+ "tokenizer.json",
+ reject_oversized = True,
+ max_bytes = _MAX_MINIMAX_TOKENIZER_BYTES,
+ ):
+ return False
+ return True
+ except (OSError, TypeError, ValueError):
+ return False
class _AudioMetadataTooLarge(ValueError):
@@ -117,16 +238,17 @@ def _read_local_audio_metadata(
filename: str,
*,
reject_oversized: bool = False,
+ max_bytes: int = _MAX_AUDIO_METADATA_BYTES,
) -> dict[str, Any]:
metadata_path = path / filename
if not metadata_path.is_file():
return {}
with metadata_path.open("rb") as handle:
- raw = handle.read(_MAX_AUDIO_METADATA_BYTES + 1)
- if len(raw) > _MAX_AUDIO_METADATA_BYTES:
+ raw = handle.read(max_bytes + 1)
+ if len(raw) > max_bytes:
if reject_oversized:
raise _AudioMetadataTooLarge(
- f"{filename} exceeds the {_MAX_AUDIO_METADATA_BYTES}-byte security inspection limit."
+ f"{filename} exceeds the {max_bytes}-byte security inspection limit."
)
return {}
value = json.loads(raw.decode("utf-8-sig"))
@@ -693,7 +815,9 @@ class NativeAudioBackend:
self.loading_models: set[str] = set()
@staticmethod
- def _token_kwargs(hf_token: Optional[str]) -> dict[str, str]:
+ def _token_kwargs(hf_token: Optional[str] | bool) -> dict[str, str | bool]:
+ if hf_token is False:
+ return {"token": False}
token = str(hf_token or "").strip()
return {"token": token} if token else {}
@@ -972,7 +1096,7 @@ class NativeAudioBackend:
from diffusers import ModularPipeline
token_kwargs = self._token_kwargs(hf_token)
- pipeline = ModularPipeline.from_pretrained(source, **token_kwargs)
+ pipeline = ModularPipeline.from_pretrained(source, trust_remote_code = False, **token_kwargs)
pipeline.load_components(
pretrained_model_name_or_path = source,
dtype = self._dtype(),
diff --git a/studio/backend/core/inference/openai_auto_download.py b/studio/backend/core/inference/openai_auto_download.py
index 49f6e872521..00579ea05c0 100644
--- a/studio/backend/core/inference/openai_auto_download.py
+++ b/studio/backend/core/inference/openai_auto_download.py
@@ -54,6 +54,9 @@ _RETRY_AFTER_S = 30
# cannot hold the slot
_FAILED_HOLD_S = 3 * _RETRY_AFTER_S
_MAX_LISTED_VARIANTS = 8
+# Probe the selected weight because speech GGUFs need not publish tokenizer sidecars.
+_REMOTE_GGUF_SPEECH_PROBE_BYTES = 32 * 1024**2
+_REMOTE_GGUF_SPEECH_PROBE_TIMEOUT_S = _CODE_PROBE_TIMEOUT_S - 2.0
@dataclass(frozen = True)
@@ -255,6 +258,27 @@ def _auth_denied(repo_id: str, hf_token: Optional[str]) -> bool:
return False
+def _probe_remote_gguf_audio_type(
+ repo_id: str, gguf_filename: str, hf_token: Optional[str], revision: Optional[str]
+) -> tuple[Optional[str], bool]:
+ try:
+ from core.inference.diffusion_compat import _read_gguf_header
+ from utils.models.gguf_metadata import classify_gguf_tts_audio_prefix
+
+ prefix = _read_gguf_header(
+ repo_id,
+ gguf_filename,
+ _hub_token(hf_token),
+ revision = revision,
+ max_bytes = _REMOTE_GGUF_SPEECH_PROBE_BYTES,
+ timeout_seconds = _REMOTE_GGUF_SPEECH_PROBE_TIMEOUT_S,
+ )
+ return classify_gguf_tts_audio_prefix(prefix) if prefix else (None, False)
+ except Exception as exc:
+ logger.debug("remote GGUF speech probe failed for %s/%s: %s", repo_id, gguf_filename, exc)
+ return None, False
+
+
def _gguf_variants(siblings, repo_id: str = "") -> dict[str, int]:
"""Quant label -> bytes the download will actually fetch.
@@ -496,6 +520,7 @@ async def maybe_auto_download(
*,
hf_token: Optional[str] = None,
require_vision: bool = False,
+ require_speech: bool = False,
subject: Optional[str] = None,
via_api_key: bool = False,
) -> Optional[AutoDownloadRefusal]:
@@ -504,9 +529,9 @@ async def maybe_auto_download(
Returns None when the request should carry on unchanged, or a refusal the
caller must raise. Only called after the local resolver has already missed.
- ``require_vision`` refuses a target with no mmproj companion rather than spend
- gigabytes on weights that cannot answer the request; the local capability guard
- only ever sees an already-downloaded model.
+ ``require_vision`` and ``require_speech`` refuse incapable targets before spending
+ gigabytes on weights; the local capability guard only ever sees an already-downloaded
+ model.
``subject`` and ``via_api_key`` describe the caller for the monitor row this
opens: the same /v1 endpoints serve Unsloth's own chat on a session JWT, so the
@@ -588,6 +613,7 @@ async def maybe_auto_download(
hf_token,
provisional,
require_vision,
+ require_speech,
subject = subject,
via_api_key = via_api_key,
)
@@ -604,6 +630,7 @@ async def _admit_and_start(
hf_token: Optional[str],
active: _Active,
require_vision: bool = False,
+ require_speech: bool = False,
*,
subject: Optional[str] = None,
via_api_key: bool = False,
@@ -737,6 +764,64 @@ async def _admit_and_start(
),
)
+ if require_speech:
+ from functools import partial
+ from utils.audio_tokens import GGUF_TTS_AUDIO_TYPES
+ from utils.models.model_config import detect_audio_type_checked
+
+ audio_type, definitive = await _bounded_probe(
+ partial(
+ detect_audio_type_checked,
+ repo_id,
+ hf_token = _hub_token(hf_token),
+ revision = getattr(info, "sha", None),
+ ),
+ timeout = _CODE_PROBE_TIMEOUT_S,
+ default = (None, False),
+ )
+ if definitive and (audio_type is None or audio_type in GGUF_TTS_AUDIO_TYPES):
+ # Prefer the selected weight; use a supported sidecar only when it is inconclusive.
+ sidecar_audio_type = audio_type
+ main_files = sorted(getattr(plan, "main_filenames", ()) or ())
+ probed_audio_type, probed_definitive = await _bounded_probe(
+ partial(
+ _probe_remote_gguf_audio_type,
+ repo_id,
+ main_files[0] if main_files else "",
+ hf_token,
+ getattr(info, "sha", None),
+ ),
+ timeout = _CODE_PROBE_TIMEOUT_S,
+ default = (None, False),
+ )
+ if probed_definitive:
+ audio_type, definitive = probed_audio_type, True
+ elif sidecar_audio_type in GGUF_TTS_AUDIO_TYPES:
+ audio_type, definitive = sidecar_audio_type, True
+ else:
+ audio_type, definitive = None, False
+ if not definitive:
+ _release(active)
+ return AutoDownloadRefusal(
+ status = 503,
+ code = "model_lookup_failed",
+ message = (
+ f"Could not verify that '{_public_label(repo_id, variant)}' supports "
+ "text-to-speech. It was not downloaded; retry shortly."
+ ),
+ retry_after = _RETRY_AFTER_S,
+ )
+ if audio_type not in GGUF_TTS_AUDIO_TYPES:
+ _release(active)
+ return AutoDownloadRefusal(
+ status = 400,
+ code = "invalid_value",
+ message = (
+ f"'{_public_label(repo_id, variant)}' is not a supported "
+ "text-to-speech GGUF model. It was not downloaded."
+ ),
+ )
+
need_bytes = _remaining_bytes(repo_id, plan, expected_bytes)
fits, free = _enough_disk(need_bytes)
if not fits:
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index e941a3ba026..ae2409d5742 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -26,7 +26,7 @@ import time
import uuid
from io import BytesIO
from pathlib import Path
-from typing import Any, Callable, Generator, Optional, Sequence, Tuple, Union
+from typing import Any, Callable, Generator, Mapping, Optional, Sequence, Tuple, Union
from core.inference.audio_device import audio_device_forces_cpu
from core.inference.native_audio import NATIVE_AUDIO_TYPES, is_native_audio_model
from core.inference.audio_errors import (
@@ -425,7 +425,11 @@ class InferenceOrchestrator:
finally:
self._top_models_ready.set()
- def _spawn_subprocess(self, config: dict) -> None:
+ def _spawn_subprocess(
+ self,
+ config: dict,
+ cache_environment: Optional[Mapping[str, str]] = None,
+ ) -> None:
"""Spawn a new inference subprocess."""
from utils.transformers_version import (
SidecarSwapInProgress,
@@ -442,7 +446,11 @@ class InferenceOrchestrator:
)
from utils.hf_cache_settings import child_environment_for_spawn, get_hf_cache_paths
- cache_env = get_hf_cache_paths().child_env({})
+ cache_env = (
+ dict(cache_environment)
+ if cache_environment is not None
+ else get_hf_cache_paths().child_env({})
+ )
with (
child_environment_for_spawn(cache_env),
@@ -1499,6 +1507,9 @@ class InferenceOrchestrator:
post_handoff_expected_free_gb: Optional[dict[int, float]] = None,
audio_device: Optional[str] = None,
on_prior_worker_released: Optional[Callable[[], None]] = None,
+ cache_environment: Optional[Mapping[str, str]] = None,
+ anonymous_hf_access: bool = False,
+ audio_codec_path: Optional[str] = None,
) -> bool:
"""Load a model for inference.
@@ -1539,6 +1550,10 @@ class InferenceOrchestrator:
# Read in the worker, which hides the accelerators before detection.
"audio_device": audio_device,
}
+ if anonymous_hf_access:
+ sub_config["anonymous_hf_access"] = True
+ if audio_codec_path is not None:
+ sub_config["audio_codec_path"] = audio_codec_path
if audio_device_forces_cpu(audio_device) and is_native_audio_model(model_name):
# Choosing a card for a load that takes none harms it twice: several
# GPUs are rejected as unsupported sharding, and required_gb becomes
@@ -1645,7 +1660,10 @@ class InferenceOrchestrator:
", xet disabled" if disable_xet else "",
)
sub_config["disable_xet"] = disable_xet
- self._spawn_subprocess(sub_config)
+ if cache_environment is None:
+ self._spawn_subprocess(sub_config)
+ else:
+ self._spawn_subprocess(sub_config, cache_environment)
# A cancel can land after the pre-spawn recheck but while _spawn_subprocess is still creating the
# queues/process. cancel_load runs off the lifecycle gate, so its _shutdown_subprocess can see _proc
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 312ccd579a2..ca5110cca93 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -157,6 +157,17 @@ def _clean_token(value: str | None) -> str | None:
return value if value and value.strip() else None
+def _config_hf_token(config: dict) -> str | bool | None:
+ if config.get("anonymous_hf_access"):
+ return False
+ return _clean_token(config.get("hf_token"))
+
+
+def _apply_worker_hf_token_environment(config: dict) -> None:
+ from hub.utils.hf_tokens import apply_token_to_child_env
+ apply_token_to_child_env(os.environ, _config_hf_token(config))
+
+
def _build_model_config(config: dict):
"""Build a ModelConfig from the config dict."""
from utils.models import ModelConfig
@@ -164,7 +175,7 @@ def _build_model_config(config: dict):
model_name = config["model_name"]
mc = ModelConfig.from_identifier(
model_id = model_name,
- hf_token = _clean_token(config.get("hf_token")),
+ hf_token = _config_hf_token(config),
gguf_variant = config.get("gguf_variant"),
)
if not mc:
@@ -386,7 +397,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
try:
mc = _build_model_config(config)
- hf_token = _clean_token(config.get("hf_token"))
+ hf_token = _config_hf_token(config)
load_in_4bit = _resolve_lora_4bit(mc, config.get("load_in_4bit", True))
# Latest-transformers sidecar models load 16-bit: bnb 4-bit feeds quantized
@@ -466,6 +477,8 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"trust_remote_code": trust_remote_code,
"gpu_ids": config.get("resolved_gpu_ids"),
}
+ if config.get("audio_codec_path") is not None:
+ load_kwargs["audio_codec_path"] = config["audio_codec_path"]
if getattr(backend, "device", None) == "mlx":
load_kwargs["parallel_mode"] = config.get("mlx_parallel_mode")
load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")
@@ -1045,6 +1058,8 @@ def run_inference_process(
here, so a generate still queued behind a cancelled one is skipped rather
than run — the cancel survives the queue handoff.
"""
+ # Apply request credentials before a Hugging Face import snapshots the environment.
+ _apply_worker_hf_token_environment(config)
os.environ["TOKENIZERS_PARALLELISM"] = "false"
os.environ["PYTHONWARNINGS"] = "ignore" # Suppress warnings at C-level before imports
@@ -1132,7 +1147,7 @@ def run_inference_process(
# Non-fatal: fall through with the installed version, but log the cause
# instead of swallowing it (issue #6103).
try:
- _activate_transformers_version(model_name, config.get("hf_token") or None)
+ _activate_transformers_version(model_name, _config_hf_token(config))
except Exception as exc:
logger.warning(
"Failed to activate transformers version for '%s' (MLX inference); "
@@ -1328,7 +1343,7 @@ def run_inference_process(
_ensure_backend_on_path()
from utils.transformers_version import _remote_lora_base, _resolve_base_model
- _hf_token = _clean_token(config.get("hf_token"))
+ _hf_token = _config_hf_token(config)
_lora_base = None
_local_adapter_cfg = Path(model_name) / "adapter_config.json"
if _local_adapter_cfg.is_file():
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 6050ae8eab0..dc402ea4ff1 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -4365,7 +4365,11 @@ class AudioSpeechRequest(BaseModel):
input: str = Field(..., min_length = 1, description = "The text to synthesize.")
model: Optional[str] = Field(
- None, description = "Model id (informational; the loaded audio model is used)."
+ None,
+ description = (
+ "Model id. A downloaded text-to-speech model named here is loaded first when "
+ "model auto-switch is on; otherwise the loaded audio model is used."
+ ),
)
voice: Optional[str] = Field(None, description = "Voice name (accepted, unused).")
response_format: Optional[str] = Field(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 3dcd77f0672..eebe2aafc52 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -49,7 +49,7 @@ import threading
import weakref
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack, asynccontextmanager, contextmanager
-from dataclasses import fields as dataclass_fields, replace
+from dataclasses import dataclass, fields as dataclass_fields, replace
import re as _re
@@ -59,6 +59,7 @@ from urllib.parse import quote as _urlquote
from utils.models import extract_model_size_b as _extract_model_size_b
from utils.api_errors import openai_error_body, anthropic_error_body, error_body_for_path
+from utils.audio_tokens import GGUF_TTS_AUDIO_TYPES as _GGUF_TTS_AUDIO_TYPES
from utils.upload_limits import STT_AUDIO_B64_MAX_CHARS, STT_AUDIO_RAW_MAX_BYTES
from hub.dependencies import get_hf_token, get_request_hf_token
from hub.utils.hf_tokens import HfTokenArg
@@ -524,6 +525,11 @@ def _tts_max_new_tokens(
return max(1, budget)
+def _speech_budget_exhausted(context_length: int, prompt_tokens: int) -> bool:
+ """Whether *prompt_tokens* leaves a context with no room to speak."""
+ return context_length - prompt_tokens - _TTS_PROMPT_FORMAT_RESERVE < _MIN_SPEECH_OUTPUT_TOKENS
+
+
def _raise_if_prompt_leaves_no_speech_budget(text: str) -> None:
"""400 when the prompt alone consumes the loaded context.
@@ -534,8 +540,7 @@ def _raise_if_prompt_leaves_no_speech_budget(text: str) -> None:
context_length = _monitor_context_length()
if not context_length:
return
- remaining = context_length - _prompt_token_estimate(text) - _TTS_PROMPT_FORMAT_RESERVE
- if remaining < _MIN_SPEECH_OUTPUT_TOKENS:
+ if _speech_budget_exhausted(context_length, _prompt_token_estimate(text)):
raise HTTPException(
status_code = 400,
detail = (
@@ -545,6 +550,18 @@ def _raise_if_prompt_leaves_no_speech_budget(text: str) -> None:
)
+_EXTRA_PROMPT_FIELD_AUDIO_TYPES = ("higgs_tts2", "moss_tts_local")
+
+
+def _speech_prompt_for_budget(audio_type: Optional[str], budget: dict) -> str:
+ text = budget.get("text") or ""
+ if audio_type not in _EXTRA_PROMPT_FIELD_AUDIO_TYPES:
+ return text
+ return _native_tts_prompt_for_budget(
+ text, audio_type, budget.get("instructions"), budget.get("language")
+ )
+
+
def _native_tts_prompt_for_budget(
text: str, audio_type: Optional[str], instructions: Optional[str], language: Optional[str]
) -> str:
@@ -576,9 +593,11 @@ def _prompt_token_estimate(prompt: str) -> int:
return count
except Exception: # noqa: BLE001 - an estimate must never fail the request
pass
- # subprocess and llama-server tokenizers are not reachable here. UTF-8 bytes are a
- # conservative upper bound for their byte-level fallbacks; under-counting can overflow
- # the loaded context, while over-counting only shortens the requested clip.
+ return _byte_fallback_prompt_tokens(prompt)
+
+
+def _byte_fallback_prompt_tokens(prompt: str) -> int:
+ # UTF-8 bytes conservatively bound the unreachable subprocess/llama-server tokenizers.
return max(1, len(prompt.encode("utf-8")))
@@ -7086,6 +7105,240 @@ def _target_accepts_request_input(
return _target_is_vision(load_path) if (needs_vision and need_image) else True
+def _resolve_target_gguf_file(load_path: str, gguf_variant: Optional[str]) -> Optional[str]:
+ from utils.models.model_config import _find_local_gguf_by_variant, detect_gguf_model
+
+ local_path = os.path.expanduser(load_path)
+ if gguf_variant and Path(local_path).is_dir():
+ return _find_local_gguf_by_variant(local_path, gguf_variant)
+ return detect_gguf_model(local_path)
+
+
+# Keep this order aligned with NativeAudioBackend._context_length.
+_NATIVE_CONTEXT_SUBCONFIGS = ("language_config", "qwen3_config", "text_config")
+_NATIVE_CONTEXT_FIELDS = (
+ "max_position_embeddings",
+ "max_sequence_length",
+ "max_seq_length",
+ "n_positions",
+ "seq_length",
+)
+
+
+def _local_config_context_length(load_path: str) -> Optional[int]:
+ import json
+
+ config_path = Path(os.path.expanduser(load_path))
+ if config_path.is_dir():
+ config_path = config_path / "config.json"
+ if not config_path.is_file():
+ return None
+ with open(config_path, encoding = "utf-8") as f:
+ config = json.load(f)
+ if not isinstance(config, dict):
+ return None
+ for candidate in (*(config.get(name) for name in _NATIVE_CONTEXT_SUBCONFIGS), config):
+ if not isinstance(candidate, dict):
+ continue
+ for field in _NATIVE_CONTEXT_FIELDS:
+ value = _positive_int_or_none(candidate.get(field))
+ if value is not None:
+ return value
+ return None
+
+
+def _target_native_context_length(
+ load_path: str,
+ is_gguf: bool,
+ gguf_variant: Optional[str] = None,
+) -> Optional[int]:
+ try:
+ if not is_gguf:
+ return _local_config_context_length(load_path)
+ from utils.models.gguf_metadata import read_gguf_context_length
+
+ gguf_file = _resolve_target_gguf_file(load_path, gguf_variant)
+ return read_gguf_context_length(gguf_file) if gguf_file else None
+ except Exception as exc:
+ logger.debug("auto-switch: context probe failed for %s: %s", load_path, exc)
+ return None
+
+
+def _target_effective_context_length(
+ load_path: str,
+ is_gguf: bool,
+ gguf_variant: Optional[str] = None,
+ override_id: Optional[str] = None,
+ audio_type: Optional[str] = None,
+ resolved_override: Optional[dict] = None,
+ override_is_resolved: bool = False,
+) -> Optional[int]:
+ # MiniMax has no measurable window; MOSS ignores saved context overrides.
+ if audio_type == "minimax_music3":
+ return None
+ if audio_type in _CONTEXT_OVERRIDE_IGNORED_AUDIO_TYPES:
+ return _target_native_context_length(load_path, is_gguf, gguf_variant)
+ try:
+ from utils.openai_auto_switch_settings import (
+ resolve_fit_max_seq_length,
+ resolve_override_for_load,
+ )
+
+ if override_is_resolved:
+ override = resolved_override
+ else:
+ _key, override = resolve_override_for_load(load_path, override_id, gguf_variant)
+ configured = _positive_int_or_none(
+ resolve_fit_max_seq_length(override, is_gguf = is_gguf) if override else None
+ )
+ if configured is None and override and is_gguf:
+ from core.inference.llama_server_args import parse_ctx_override
+ configured = _positive_int_or_none(parse_ctx_override(override.get("llama_extra_args")))
+ if configured is not None:
+ from core.inference.native_audio import NATIVE_AUDIO_TYPES
+ if not is_gguf and audio_type in NATIVE_AUDIO_TYPES:
+ detected = _target_native_context_length(load_path, is_gguf, gguf_variant)
+ return min(configured, detected) if detected else configured
+ return configured
+ except Exception as exc:
+ logger.debug("auto-switch: context override lookup failed for %s: %s", load_path, exc)
+ from core.inference.native_audio import NATIVE_AUDIO_TYPES
+
+ if not is_gguf and audio_type not in NATIVE_AUDIO_TYPES:
+ # The generic loader converts its zero default to 2048, not the declared window.
+ return _STANDARD_LOAD_DEFAULT_CONTEXT
+ return _target_native_context_length(load_path, is_gguf, gguf_variant)
+
+
+def _target_speech_audio_type(
+ load_path: str,
+ is_gguf: bool,
+ gguf_variant: Optional[str] = None,
+) -> Optional[str]:
+ from core.inference.local_model_resolver import _host_serves_mlx
+ from utils.models.model_config import (
+ _find_local_gguf_by_variant,
+ detect_audio_type,
+ detect_gguf_model,
+ )
+ try:
+ if not is_gguf:
+ from core.inference.native_audio import (
+ REMOTE_CODE_AUDIO_TYPES,
+ is_native_audio_model,
+ )
+
+ # Native audio precedes MLX and has an MPS path; ordinary codecs cannot serve there.
+ if _host_serves_mlx() and not is_native_audio_model(load_path):
+ return None
+ audio_type = detect_audio_type(
+ load_path, hf_token = os.environ.get("HF_TOKEN"), local_files_only = True
+ )
+ # Auto-switch never grants trust_remote_code; reject before evicting the resident.
+ if audio_type in REMOTE_CODE_AUDIO_TYPES:
+ return None
+ return audio_type if audio_type in _TRANSFORMERS_TTS_AUDIO_TYPES else None
+ from utils.models.gguf_metadata import read_gguf_tts_audio_type
+
+ gguf_file = _resolve_target_gguf_file(load_path, gguf_variant)
+ audio_type = read_gguf_tts_audio_type(gguf_file) if gguf_file else None
+ return audio_type if audio_type in _GGUF_TTS_AUDIO_TYPES else None
+ except Exception as exc:
+ logger.debug("auto-switch: speech probe failed for %s: %s", load_path, exc)
+ return None
+
+
+@dataclass(frozen = True)
+class _SpeechCodecPreflightResult:
+ cache_environment: Optional[dict[str, str]] = None
+ codec_path: Optional[str] = None
+
+
+def _preflight_speech_codec_for_switch(
+ audio_type: str,
+ load_path: str,
+ is_gguf: bool,
+ hf_token: Optional[str] = None,
+) -> _SpeechCodecPreflightResult:
+ from utils.utils import hf_env_offline
+
+ offline = hf_env_offline()
+ hub_token = hf_token or False
+ if audio_type == "snac":
+ from huggingface_hub import snapshot_download
+ from utils.hf_cache_settings import get_hf_cache_paths
+
+ cache_paths = get_hf_cache_paths()
+ staged = snapshot_download(
+ "hubertsiuzdak/snac_24khz",
+ token = hub_token,
+ cache_dir = str(cache_paths.hub_cache),
+ local_files_only = offline,
+ )
+ return _SpeechCodecPreflightResult(cache_paths.child_env({}), str(staged))
+ elif audio_type == "bicodec":
+ from core.inference.audio_codecs import resolve_bicodec_repo_path
+ from utils.hf_cache_settings import get_hf_cache_paths
+
+ cache_paths = get_hf_cache_paths()
+ staged = resolve_bicodec_repo_path(
+ None if is_gguf else load_path,
+ hf_token = hub_token,
+ local_files_only = offline,
+ cache_dir = str(cache_paths.hub_cache),
+ )
+ return _SpeechCodecPreflightResult(cache_paths.child_env({}), str(staged))
+ elif audio_type == "dac":
+ from utils.third_party_source import ensure_dac_speech_weights, ensure_outetts_source
+ from utils.hf_cache_settings import get_hf_cache_paths
+
+ cache_paths = get_hf_cache_paths()
+ ensure_outetts_source()
+ staged = ensure_dac_speech_weights(
+ hub_cache = cache_paths.hub_cache,
+ hf_token = hub_token,
+ )
+ return _SpeechCodecPreflightResult(cache_paths.child_env({}), str(staged))
+ elif audio_type == "higgs_tts2":
+ from core.inference.native_audio import (
+ higgs_tts2_codec_local_complete,
+ native_audio_security_targets,
+ )
+ from huggingface_hub import snapshot_download
+ from utils.hf_cache_settings import get_hf_cache_paths
+
+ cache_paths = get_hf_cache_paths()
+ companions = native_audio_security_targets(load_path, audio_type, hub_token)[1:]
+ if not companions:
+ raise RuntimeError("Higgs TTS 2 exposes no companion audio tokenizer.")
+ for companion in companions:
+ local_companion = Path(companion).expanduser()
+ staged = (
+ str(local_companion)
+ if local_companion.exists()
+ else snapshot_download(
+ companion,
+ token = hub_token,
+ cache_dir = str(cache_paths.hub_cache),
+ local_files_only = offline,
+ )
+ )
+ from utils.security import evaluate_file_security
+
+ security = evaluate_file_security(
+ staged,
+ hf_token = hub_token,
+ local_only_load = offline,
+ )
+ if security.blocked:
+ raise RuntimeError(security.reason)
+ if not higgs_tts2_codec_local_complete(staged):
+ raise RuntimeError(f"Higgs TTS 2 companion '{companion}' is incomplete.")
+ # Pin the verified cache across the lifecycle wait; settings may change meanwhile.
+ return _SpeechCodecPreflightResult(cache_paths.child_env({}))
+ return _SpeechCodecPreflightResult()
+
+
_AUDIO_IMAGE_INPUT_DETAIL = (
"This model takes audio or an image in one message, not both. Send the image on its own turn."
)
@@ -7433,6 +7686,7 @@ async def _maybe_auto_download_model(
fastapi_request: Optional[Request],
*,
require_vision: bool = False,
+ require_speech: bool = False,
current_subject: Optional[str] = None,
) -> None:
"""Opt-in: start fetching a named GGUF this server doesn't have.
@@ -7456,6 +7710,7 @@ async def _maybe_auto_download_model(
requested_model,
hf_token = _auto_download_hf_token(fastapi_request),
require_vision = require_vision,
+ require_speech = require_speech,
subject = current_subject,
# These endpoints also serve Unsloth's chat on a JWT, so only mark real API traffic.
via_api_key = _request_used_api_key(fastapi_request),
@@ -7926,6 +8181,8 @@ async def _maybe_auto_switch_model(
gguf_only: bool = False,
audio_preflight: Optional[dict] = None,
image_preflight: Optional[dict] = None,
+ require_speech: bool = False,
+ speech_budget: Optional[dict] = None,
) -> None:
"""Load a downloaded local model named by an OpenAI request when auto-switch is on.
@@ -7978,6 +8235,7 @@ async def _maybe_auto_switch_model(
"generation_cancel_event",
None,
)
+ caller_hf_token = _auto_download_hf_token(fastapi_request)
def _raise_if_generation_cancelled() -> None:
if generation_cancel_event is not None and generation_cancel_event.is_set():
@@ -8095,6 +8353,7 @@ async def _maybe_auto_switch_model(
# GGUF carries both from one mmproj, so the download guard takes
# either need; splitting them here would fetch a text-only repo.
require_vision = require_vision or require_audio_input,
+ require_speech = require_speech,
current_subject = current_subject,
)
# Idle-unload may have freed the model; reload exactly what it freed
@@ -8252,6 +8511,70 @@ async def _maybe_auto_switch_model(
param = "model",
),
)
+ speech_type = None
+ if require_speech and resolved is not None:
+ speech_type = await asyncio.to_thread(
+ _target_speech_audio_type, target_id, target_is_gguf, variant
+ )
+ if speech_type is None:
+ raise HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ "The requested model is not a text-to-speech model.",
+ status = 400,
+ code = "invalid_value",
+ param = "model",
+ ),
+ )
+ from core.inference.native_audio import PYTHON310_AUDIO_TYPES
+
+ if speech_type in PYTHON310_AUDIO_TYPES and sys.version_info < (3, 10):
+ raise HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ "The requested text-to-speech model requires Python 3.10 or newer in Studio.",
+ status = 400,
+ code = "unsupported_runtime",
+ param = "model",
+ ),
+ )
+ if speech_budget is not None:
+ target_context = await asyncio.to_thread(
+ _target_effective_context_length,
+ target_id,
+ target_is_gguf,
+ variant,
+ override_id,
+ speech_type,
+ )
+ prompt_tokens = _byte_fallback_prompt_tokens(
+ _speech_prompt_for_budget(speech_type, speech_budget)
+ )
+ if target_context and _speech_budget_exhausted(target_context, prompt_tokens):
+ raise HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ f"Input is too long for the requested model's "
+ f"{target_context}-token context. Shorten it, or pick a model "
+ "with a larger context.",
+ status = 400,
+ code = "invalid_value",
+ param = "input",
+ ),
+ )
+ if (
+ speech_type == "minimax_music3"
+ and not str((speech_budget or {}).get("instructions") or "").strip()
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ _MINIMAX_NEEDS_DESCRIPTION,
+ status = 400,
+ code = "invalid_value",
+ param = "instructions",
+ ),
+ )
# resolver branch only, like the probe above: a reload-stash restore changes no format.
if audio_preflight is not None and resolved is not None:
await _preflight_audio_for_switch(audio_preflight, target_is_gguf)
@@ -8261,6 +8584,34 @@ async def _maybe_auto_switch_model(
and (target_is_gguf or not require_audio_input)
):
await _preflight_image_for_switch(image_preflight, target_is_gguf)
+ speech_cache_environment = None
+ speech_codec_path = None
+ if speech_type is not None:
+ try:
+ speech_preflight_result = await asyncio.to_thread(
+ _preflight_speech_codec_for_switch,
+ speech_type,
+ target_id,
+ target_is_gguf,
+ caller_hf_token,
+ )
+ if isinstance(speech_preflight_result, _SpeechCodecPreflightResult):
+ speech_cache_environment = speech_preflight_result.cache_environment
+ speech_codec_path = speech_preflight_result.codec_path
+ elif isinstance(speech_preflight_result, dict):
+ speech_cache_environment = speech_preflight_result
+ except Exception:
+ raise HTTPException(
+ status_code = 503,
+ detail = openai_error_body(
+ "The requested model's codec assets are unavailable. Connect this "
+ "server to the network once to download them, or install them in the "
+ "active cache before retrying.",
+ status = 503,
+ code = "codec_unavailable",
+ param = "model",
+ ),
+ ) from None
key = _switch_key(override_id, variant)
_note_switch_waiter(key, 1)
waiter_noted = True
@@ -8296,10 +8647,40 @@ async def _maybe_auto_switch_model(
_override_key, override = resolve_override_for_load(
target_id, override_id, variant
)
+ if require_speech and resolved is not None and speech_budget is not None:
+ target_context = await asyncio.to_thread(
+ _target_effective_context_length,
+ target_id,
+ target_is_gguf,
+ variant,
+ override_id,
+ speech_type,
+ override,
+ True,
+ )
+ prompt_tokens = _byte_fallback_prompt_tokens(
+ _speech_prompt_for_budget(speech_type, speech_budget)
+ )
+ if target_context and _speech_budget_exhausted(
+ target_context, prompt_tokens
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ f"Input is too long for the requested model's "
+ f"{target_context}-token context. Shorten it, or pick a model "
+ "with a larger context.",
+ status = 400,
+ code = "invalid_value",
+ param = "input",
+ ),
+ )
load_kwargs = {"model_path": target_id, "gguf_variant": variant}
load_kwargs.update(
model_override_load_kwargs(override, is_gguf = target_is_gguf)
)
+ if caller_hf_token:
+ load_kwargs["hf_token"] = caller_hf_token
saved_gpu_ids = load_kwargs.get("gpu_ids")
if saved_gpu_ids and not await _override_gpu_ids_still_resolve(
saved_gpu_ids, stored_gpu_index_kind(override)
@@ -8332,13 +8713,20 @@ async def _maybe_auto_switch_model(
if generation_cancel_event is not None
else {}
)
+ load_internal_kw = dict(durable_cancel_kw)
+ if speech_cache_environment is not None:
+ load_internal_kw["cache_environment"] = speech_cache_environment
+ if speech_codec_path is not None:
+ load_internal_kw["speech_codec_path"] = speech_codec_path
+ if speech_type is not None and caller_hf_token is None:
+ load_internal_kw["anonymous_hf_access"] = True
try:
await _load_model_impl(
LoadRequest(**load_kwargs),
fastapi_request,
current_subject,
current_request_counted = True,
- **durable_cancel_kw,
+ **load_internal_kw,
)
except HTTPException as exc:
# The pre-flight check cannot mirror every loader gpu_ids rule,
@@ -8361,7 +8749,7 @@ async def _maybe_auto_switch_model(
fastapi_request,
current_subject,
current_request_counted = True,
- **durable_cancel_kw,
+ **load_internal_kw,
)
_switch_loaded_ok = True
# publish the completed load before a late cancellation is observed.
@@ -13330,6 +13718,9 @@ async def _load_model_impl(
on_reload_confirmed = None,
allow_gpu_owner_eviction: bool = True,
load_cancel_event: Optional[threading.Event] = None,
+ cache_environment: Optional[dict[str, str]] = None,
+ anonymous_hf_access: bool = False,
+ speech_codec_path: Optional[str] = None,
):
from core.inference.llama_cpp import LlamaServerNotFoundError
@@ -13669,6 +14060,8 @@ async def _load_model_impl(
placement = placement,
n_parallel = _n_parallel,
)
+ if speech_codec_path is not None:
+ gguf_intent = replace(gguf_intent, audio_codec_path = speech_codec_path)
same_loaded_model = llama_backend.matches_load_source(gguf_intent)
if same_loaded_model and config.gguf_hf_repo and llama_backend.gguf_path:
gguf_intent = replace(
@@ -14175,6 +14568,10 @@ async def _load_model_impl(
# claim is all that stops a second pipeline allocating over a resident model).
# load_model fires it in between; the post-load release covers a re-taken claim.
_release_chat_after_teardown = (lambda: release(CHAT)) if not chat_load_needs_gpu else None
+ anonymous_hf_kw = {"anonymous_hf_access": True} if anonymous_hf_access else {}
+ speech_codec_kw = (
+ {"audio_codec_path": speech_codec_path} if speech_codec_path is not None else {}
+ )
try:
success = await asyncio.to_thread(
backend.load_model,
@@ -14196,6 +14593,9 @@ async def _load_model_impl(
on_prior_worker_released = _release_chat_after_teardown,
post_handoff_expected_free_gb = post_chat_handoff_expected_free_gb,
audio_device = request.audio_device,
+ cache_environment = cache_environment,
+ **anonymous_hf_kw,
+ **speech_codec_kw,
)
except Exception:
_restore_marker_if_prior_preview_still_resident()
@@ -16551,7 +16951,11 @@ _TRANSFORMERS_TTS_AUDIO_TYPES = frozenset(
"minimax_music3",
)
)
-_GGUF_TTS_AUDIO_TYPES = frozenset(("snac", "bicodec", "dac"))
+# NativeAudioBackend._context_length ignores the requested window for these.
+_CONTEXT_OVERRIDE_IGNORED_AUDIO_TYPES = frozenset(("moss_tts_local", "moss_tts_nano"))
+# inference.py: load_model raises max_seq_length <= 0 to this before loading.
+_STANDARD_LOAD_DEFAULT_CONTEXT = 2048
+_MINIMAX_NEEDS_DESCRIPTION = "MiniMax Music 3 requires a music description in addition to lyrics."
async def _generate_tts_wav(
@@ -16561,23 +16965,28 @@ async def _generate_tts_wav(
current_subject: str,
*,
speech_api_default_max_tokens: bool = False,
+ requested_model: str = _RELOAD_ONLY_MODEL,
) -> tuple[bytes, int, str, Optional[str]]:
"""Shared core of /audio/generate and /audio/speech. Returns
(wav_bytes, sample_rate, model_name, audio_type)."""
- _raise_if_prompt_leaves_no_speech_budget(text)
- # Restore an idle-evicted GGUF before selecting a backend: this path is
- # keep-warm-tracked but had no reload hook, so a standalone idle TTL could
- # unload an audio GGUF the next request then failed to restore. Validation
- # above ran first, so an invalid request never triggers a reload.
- #
- # Reload-only on purpose: a local GGUF's audio-input capability is not a cheap
- # pre-load probe (the companion mmproj signal can't tell an audio projector
- # from a vision one, and codec-based TTS ships no projector at all), so passing
- # the client model through the resolver could load a text- or vision-only target
- # and evict the working audio model before the audio backend check fails. Only
- # the idle-stash restore runs here; switching TTS models is an explicit /load.
+ # A named target must be budgeted against its own context after preflight.
+ if requested_model == _RELOAD_ONLY_MODEL:
+ _raise_if_prompt_leaves_no_speech_budget(text)
await _maybe_auto_switch_model(
- _RELOAD_ONLY_MODEL, request, current_subject, claim_resident = False
+ requested_model,
+ request,
+ current_subject,
+ claim_resident = False,
+ require_speech = True,
+ speech_budget = (
+ None
+ if requested_model == _RELOAD_ONLY_MODEL
+ else {
+ "text": text,
+ "instructions": payload.audio_instructions,
+ "language": payload.audio_language,
+ }
+ ),
)
# Again, now that a context exists to measure against. The check above runs before the
# restore so an invalid request never triggers a reload, but with nothing loaded it has
@@ -16655,11 +17064,8 @@ async def _generate_tts_wav(
detail = f"Active model does not support text-to-speech (audio_type={audio_type or 'unknown'}).",
)
if audio_type == "minimax_music3" and not str(payload.audio_instructions or "").strip():
- raise HTTPException(
- status_code = 400,
- detail = "MiniMax Music 3 requires a music description in addition to lyrics.",
- )
- if audio_type in ("higgs_tts2", "moss_tts_local"):
+ raise HTTPException(status_code = 400, detail = _MINIMAX_NEEDS_DESCRIPTION)
+ if audio_type in _EXTRA_PROMPT_FIELD_AUDIO_TYPES:
prompt_for_budget = _native_tts_prompt_for_budget(
text,
audio_type,
@@ -16798,7 +17204,14 @@ async def generate_audio(
text = last_user_msg["content"]
wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(
- text, payload, request, current_subject
+ text,
+ payload,
+ request,
+ current_subject,
+ # ``or`` like /v1/audio/speech: an explicitly empty model is accepted by the
+ # request model, and without this it stops the hook at its falsey check before
+ # the idle-stash restore, failing a request the sibling route serves.
+ requested_model = _switch_model_for_payload(payload) or _RELOAD_ONLY_MODEL,
)
persisted_clip = await asyncio.to_thread(
_persist_tts_clip, wav_bytes, sample_rate, text, model_name, audio_type
@@ -16939,10 +17352,10 @@ async def openai_audio_speech(
) -> Response:
"""OpenAI-compatible text-to-speech (POST /v1/audio/speech).
- With ``provider_id`` the request is proxied to that connection, forwarding
- model/voice/speed/instructions. Otherwise the loaded model is used: ``model`` is informational,
- ``voice``/``speed`` ignored, and only WAV exists, so another ``response_format`` is
- a 400 rather than a silent container mismatch."""
+ With ``provider_id`` the request is proxied. Otherwise an omitted ``model`` serves the
+ resident model. A named downloaded local model is loaded when auto-switch is on; when
+ it is off, the shared switch guard rejects a different recognized local model.
+ Local ``voice``/``speed`` are ignored and only WAV is supported."""
if body.provider_id:
fmt = (body.response_format or "wav").strip().lower()
if fmt != "wav":
@@ -17006,6 +17419,7 @@ async def openai_audio_speech(
request,
current_subject,
speech_api_default_max_tokens = body.max_new_tokens is None,
+ requested_model = body.model or _RELOAD_ONLY_MODEL,
)
api_monitor.relabel(monitor_id, model_name)
await asyncio.to_thread(
diff --git a/studio/backend/tests/test_audio_probe_target.py b/studio/backend/tests/test_audio_probe_target.py
index 111c278d6a0..aa2629f6fef 100644
--- a/studio/backend/tests/test_audio_probe_target.py
+++ b/studio/backend/tests/test_audio_probe_target.py
@@ -42,12 +42,12 @@ def test_the_merged_export_load_path_resolves_the_alias_the_same_way():
"Spark-TTS-0.5B/LLM" -> "unsloth/Spark-TTS-0.5B" mapping; it now shares load_scan_target
with the capability probe here and with the trainer preflight in routes/training.py, so
the three cannot drift."""
- # Read rather than import: core.inference.inference pulls the whole Unsloth stack,
+ # Read rather than import: the codec module pulls optional audio dependencies in,
# which is what made a second, dependency-light copy of this mapping tempting.
from pathlib import Path
source = (
- Path(__file__).resolve().parents[1] / "core" / "inference" / "inference.py"
+ Path(__file__).resolve().parents[1] / "core" / "inference" / "audio_codecs.py"
).read_text(encoding = "utf-8")
assert "load_scan_target(" in source
assert "spark_tts_base_repo" not in source
diff --git a/studio/backend/tests/test_audio_token_detection.py b/studio/backend/tests/test_audio_token_detection.py
index 8685286faf2..cf9fcf78a57 100644
--- a/studio/backend/tests/test_audio_token_detection.py
+++ b/studio/backend/tests/test_audio_token_detection.py
@@ -7,6 +7,7 @@
from __future__ import annotations
import json
+import os
import pytest
from utils.audio_tokens import AUDIO_TOKEN_PATTERNS
@@ -36,6 +37,25 @@ def test_local_native_audio_model_type_is_detected(tmp_path):
assert detect_audio_type_checked(str(tmp_path)) == ("moss_tts_nano", True)
+def test_replacing_local_audio_metadata_invalidates_the_cached_verdict(tmp_path, monkeypatch):
+ from utils.models import model_config
+
+ monkeypatch.setattr(model_config, "_audio_detection_cache", {})
+ config = tmp_path / "config.json"
+ replacement = tmp_path / "replacement.json"
+ config.write_text('{"model_type":"moss_tts_nano"}', encoding = "utf-8")
+ replacement.write_text('{"model_type":"not_audio_xyz"}', encoding = "utf-8")
+ assert config.stat().st_size == replacement.stat().st_size
+ assert model_config.detect_audio_type_checked(str(tmp_path))[0] == "moss_tts_nano"
+
+ original = config.stat()
+ os.replace(replacement, config)
+ os.utime(config, ns = (config.stat().st_atime_ns, original.st_mtime_ns))
+ assert config.stat().st_size == original.st_size
+ assert config.stat().st_mtime_ns == original.st_mtime_ns
+ assert model_config.detect_audio_type_checked(str(tmp_path))[0] is None
+
+
def _classify(tokens: list[str]) -> str | None:
"""Mirror _check_token_patterns: first match in dict order wins."""
for audio_type, check in AUDIO_TOKEN_PATTERNS.items():
diff --git a/studio/backend/tests/test_gguf_tts_audio_type.py b/studio/backend/tests/test_gguf_tts_audio_type.py
new file mode 100644
index 00000000000..21a6b21510c
--- /dev/null
+++ b/studio/backend/tests/test_gguf_tts_audio_type.py
@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import os
+import struct
+from types import SimpleNamespace
+
+import pytest
+
+from utils.audio_tokens import SNAC_PROBE_TOKEN_IDS
+from utils.models import gguf_metadata
+from utils.models.gguf_metadata import classify_gguf_tts_audio_prefix, read_gguf_tts_audio_type
+
+
+def test_outetts_v3_speakerless_prompt_stops_before_generated_features():
+ from core.inference.chat_template_helpers import build_dac_tts_prompt
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ expected = "<|im_start|>\n<|text_start|>Read this aloud.<|text_end|>\n<|audio_start|>\n"
+
+ assert build_dac_tts_prompt("Read this aloud.") == expected
+ assert LlamaCppBackend._TTS_PROMPTS["dac"][0].format(text = "Read this aloud.") == expected
+ assert "<|global_features_start|>" not in expected
+
+
+def test_native_outetts_uses_the_same_speakerless_prompt(monkeypatch):
+ pytest.importorskip("peft")
+ import torch
+ from core.inference.inference import InferenceBackend
+
+ captured = []
+
+ def tokenizer(prompts, **_kwargs):
+ captured.extend(prompts)
+ raise RuntimeError("prompt captured")
+
+ backend = InferenceBackend.__new__(InferenceBackend)
+ backend._audio_codec_manager = SimpleNamespace()
+ monkeypatch.setattr(backend, "_patch_repetition_penalty_processor", lambda: None)
+ with pytest.raises(RuntimeError, match = "prompt captured"):
+ backend._generate_dac(
+ SimpleNamespace(device = "cpu", dtype = torch.float32),
+ tokenizer,
+ "Read this aloud.",
+ temperature = 0.6,
+ top_k = 50,
+ top_p = 0.95,
+ min_p = 0.0,
+ max_new_tokens = 64,
+ repetition_penalty = 1.1,
+ )
+ assert captured == [
+ "<|im_start|>\n<|text_start|>Read this aloud.<|text_end|>\n<|audio_start|>\n"
+ ]
+
+
+def _string(value: str) -> bytes:
+ data = value.encode()
+ return struct.pack("<Q", len(data)) + data
+
+
+def _write_gguf(
+ path,
+ tokens,
+ token_types = None,
+):
+ metadata = _string("general.architecture") + struct.pack("<I", 8) + _string("llama")
+ if tokens is not None:
+ array = struct.pack("<IQ", 8, len(tokens)) + b"".join(_string(t) for t in tokens)
+ metadata += _string("tokenizer.ggml.tokens") + struct.pack("<I", 9) + array
+ token_types = token_types or [3] * len(tokens)
+ types = struct.pack("<IQ", 5, len(token_types)) + struct.pack(
+ f"<{len(token_types)}i", *token_types
+ )
+ metadata += _string("tokenizer.ggml.token_type") + struct.pack("<I", 9) + types
+ kv_count = 1 + (2 if tokens is not None else 0)
+ path.write_bytes(struct.pack("<IIQQ", 0x46554747, 3, 0, kv_count) + metadata)
+ return str(path)
+
+
+def _snac_vocab():
+ """Orpheus's shape: the codes start where the base vocabulary ends, so the two ids
+ the serving detector detokenizes land on codes."""
+ base = [f"t{i}" for i in range(SNAC_PROBE_TOKEN_IDS[0])]
+ return [*base, *(f"<custom_token_{i}>" for i in range(10_002))]
+
+
[email protected](autouse = True)
+def _clear_cache():
+ gguf_metadata._TTS_AUDIO_TYPE_CACHE.clear()
+
+
+def test_atomic_replacement_invalidates_a_same_size_same_mtime_verdict(tmp_path):
+ path = tmp_path / "model.gguf"
+ replacement = tmp_path / "replacement.gguf"
+ _write_gguf(path, ["<|bicodec_semantic_0|>", "<|bicodec_global_0|>"])
+ _write_gguf(replacement, ["x" * 22, "y" * 20])
+ assert path.stat().st_size == replacement.stat().st_size
+ assert read_gguf_tts_audio_type(str(path)) == "bicodec"
+ original = path.stat()
+ os.replace(replacement, path)
+ os.utime(path, ns = (path.stat().st_atime_ns, original.st_mtime_ns))
+ assert path.stat().st_size == original.st_size
+ assert path.stat().st_mtime_ns == original.st_mtime_ns
+ assert read_gguf_tts_audio_type(str(path)) is None
+
+
+ ("key", "element_type"),
+ (("tokenizer.ggml.tokens", 8), ("tokenizer.ggml.token_type", 5)),
+)
+def test_oversized_vocabulary_arrays_are_rejected_before_allocation(tmp_path, key, element_type):
+ path = tmp_path / f"oversized-{element_type}.gguf"
+ metadata = (
+ _string(key)
+ + struct.pack("<I", 9)
+ + struct.pack("<IQ", element_type, gguf_metadata._MAX_GGUF_VOCAB_ENTRIES + 1)
+ )
+ path.write_bytes(struct.pack("<IIQQ", 0x46554747, 3, 0, 1) + metadata)
+ assert gguf_metadata._parse_gguf_marker_tokens(str(path)) is None
+
+
+def test_the_switch_probe_reads_the_variant_the_load_will_open(tmp_path, monkeypatch):
+ import routes.inference as inference_route
+ from utils.models import model_config
+
+ speech = _write_gguf(
+ tmp_path / "model-Q8_0.gguf", ["<|bicodec_semantic_0|>", "<|bicodec_global_0|>"]
+ )
+ text = _write_gguf(tmp_path / "model-Q4_K_M.gguf", ["<|im_start|>"])
+ monkeypatch.setattr(
+ model_config,
+ "_find_local_gguf_by_variant",
+ lambda _d, variant, *_a: speech if variant == "Q8_0" else text,
+ )
+ assert inference_route._target_speech_audio_type(str(tmp_path), True, "Q8_0") == "bicodec"
+ assert inference_route._target_speech_audio_type(str(tmp_path), True, "Q4_K_M") is None
+ monkeypatch.setattr(model_config, "detect_gguf_model", lambda _p, *_a: speech)
+ assert inference_route._target_speech_audio_type(speech, True) == "bicodec"
+
+
+ "tokens,types,expected",
+ [
+ (["<|bicodec_semantic_0|>", "<|bicodec_global_0|>"], None, "bicodec"),
+ (["<|c1_0|>", "<|c2_0|>"], None, "dac"),
+ (["<|c1_0|>", "<|c2_0|>"], [1, 1], None),
+ (["<|bicodec_semantic_0|>"], None, None),
+ (["<|audio|>", "<|c1_0|>", "<|c2_0|>"], None, None),
+ (["<|audio_start|>", "<|audio_end|>"], None, None),
+ (["hello", "<|im_start|>"], None, None),
+ (None, None, None),
+ ],
+)
+def test_gguf_speech_vocabulary(tmp_path, tokens, types, expected):
+ path = _write_gguf(tmp_path / "model.gguf", tokens, types)
+ assert read_gguf_tts_audio_type(path) == expected
+
+
[email protected]("offset,expected", [(0, "snac"), (8, None)])
+def test_snac_requires_runtime_probe_positions(tmp_path, offset, expected):
+ assert (
+ read_gguf_tts_audio_type(
+ _write_gguf(tmp_path / "snac.gguf", ["pad"] * offset + _snac_vocab())
+ )
+ == expected
+ )
+
+
+def test_remote_prefix_rejects_truncation(tmp_path):
+ from pathlib import Path
+
+ path = _write_gguf(tmp_path / "model.gguf", ["<|c1_0|>", "<|c2_0|>"])
+ data = Path(path).read_bytes()
+ assert classify_gguf_tts_audio_prefix(data) == ("dac", True)
+ assert classify_gguf_tts_audio_prefix(data[:-1]) == (None, False)
diff --git a/studio/backend/tests/test_native_audio.py b/studio/backend/tests/test_native_audio.py
index d79db68f554..b86f3bd6471 100644
--- a/studio/backend/tests/test_native_audio.py
+++ b/studio/backend/tests/test_native_audio.py
@@ -43,6 +43,15 @@ def _backend(audio_type: str, **entry):
return backend
+def test_anonymous_worker_token_cannot_fall_back_to_the_host_login():
+ from core.inference.inference import _hf_token_for_loader
+ from core.inference.worker import _config_hf_token
+
+ assert _config_hf_token({"hf_token": "", "anonymous_hf_access": True}) is False
+ assert _hf_token_for_loader(False) is False
+ assert NativeAudioBackend._token_kwargs(False) == {"token": False}
+
+
@pytest.mark.parametrize(
("repo", "companion"),
(
@@ -620,8 +629,9 @@ def test_minimax_loader_resolves_components_from_the_selected_checkpoint(monkeyp
pipeline = Pipeline()
- def from_pretrained(source, **_kwargs):
+ def from_pretrained(source, **kwargs):
seen["source"] = source
+ seen["from_pretrained"] = kwargs
return pipeline
monkeypatch.setitem(
@@ -635,6 +645,7 @@ def test_minimax_loader_resolves_components_from_the_selected_checkpoint(monkeyp
entry = {}
backend._load_minimax_music3(entry, "/models/minimax-custom", None)
+ assert seen["from_pretrained"]["trust_remote_code"] is False
assert seen["components"]["pretrained_model_name_or_path"] == "/models/minimax-custom"
assert seen["device"] == "cuda"
assert entry["pipeline"] is pipeline
diff --git a/studio/backend/tests/test_openai_audio_speech_route.py b/studio/backend/tests/test_openai_audio_speech_route.py
index 6203b3d8cc9..4264a2d68de 100644
--- a/studio/backend/tests/test_openai_audio_speech_route.py
+++ b/studio/backend/tests/test_openai_audio_speech_route.py
@@ -8,8 +8,10 @@ gallery persistence and the raw-WAV response without torch, weights or a GPU."""
from __future__ import annotations
+import asyncio
import json
from contextlib import asynccontextmanager
+from types import SimpleNamespace
import pytest
from fastapi import FastAPI, HTTPException, Request
@@ -28,8 +30,8 @@ _WAV = b"RIFF\x24\x00\x00\x00WAVEfmt fake-payload"
def _make_client(monkeypatch, generate = None):
calls = []
- async def _fake_generate(text, payload, request, current_subject, **_kwargs):
- calls.append({"text": text, "payload": payload})
+ async def _fake_generate(text, payload, request, current_subject, **kwargs):
+ calls.append({"text": text, "payload": payload, **kwargs})
if generate is not None:
return await generate(text)
return _WAV, 24000, "unsloth/orpheus-3b-0.1-ft", "snac"
@@ -197,6 +199,39 @@ def test_an_over_context_prompt_is_a_client_error(monkeypatch):
routes_module._raise_if_prompt_leaves_no_speech_budget("A short line.")
[email protected]("model", [None, "", "org/B-GGUF"])
+def test_speech_model_selection(monkeypatch, model):
+ cli, calls, _saved = _make_client(monkeypatch)
+ body = {"input": "hi", **({"model": model} if model is not None else {})}
+ assert cli.post("/v1/audio/speech", json = body).status_code == 200
+ assert calls[0]["requested_model"] == (model or routes_module._RELOAD_ONLY_MODEL)
+
+
[email protected]("named", [False, True])
+def test_only_resident_requests_use_the_pre_switch_budget(monkeypatch, named):
+ async def _switch(_model, *_a, **kw):
+ assert kw["require_speech"] is True
+ raise RuntimeError("reached the switch")
+
+ monkeypatch.setattr(routes_module, "_maybe_auto_switch_model", _switch)
+ monkeypatch.setattr(routes_module, "_monitor_context_length", lambda: 2048)
+ monkeypatch.setattr(routes_module, "_prompt_token_estimate", lambda _t: 2048)
+ payload = SimpleNamespace(audio_instructions = None, audio_language = None)
+ request = SimpleNamespace(state = SimpleNamespace(skip_api_monitor = True))
+ model = "org/B-GGUF" if named else routes_module._RELOAD_ONLY_MODEL
+ with pytest.raises(RuntimeError if named else HTTPException) as error:
+ asyncio.run(
+ routes_module._generate_tts_wav(
+ "a long line",
+ payload,
+ request,
+ "tester",
+ requested_model = model,
+ )
+ )
+ assert str(error.value) == "reached the switch" if named else error.value.status_code == 400
+
+
def test_the_shared_core_guards_before_generating():
"""Wired in _generate_tts_wav so /audio/generate inherits it, not only /audio/speech."""
import inspect
@@ -219,9 +254,7 @@ def test_the_budget_is_rechecked_after_an_idle_model_is_restored():
if "_raise_if_prompt_leaves_no_speech_budget(text)" in line
]
restore = next(
- i
- for i, line in enumerate(source.splitlines())
- if "_RELOAD_ONLY_MODEL, request, current_subject" in line
+ i for i, line in enumerate(source.splitlines()) if "await _maybe_auto_switch_model(" in line
)
assert len(guards) == 2, "one check before the restore, one after"
assert guards[0] < restore < guards[1]
diff --git a/studio/backend/tests/test_openai_auto_download.py b/studio/backend/tests/test_openai_auto_download.py
index c206f562b7a..a4fa872ebf5 100644
--- a/studio/backend/tests/test_openai_auto_download.py
+++ b/studio/backend/tests/test_openai_auto_download.py
@@ -2047,3 +2047,35 @@ def test_an_exact_generic_variant_beats_the_default_pick(hub):
hub["info"] = _gguf_repo_info()
assert _run("unsloth/x-GGUF:Q2_K").status == 404
assert hub["started"] == []
+
+
+ "sidecar,probe,code",
+ [
+ ((None, True), (None, True), "invalid_value"),
+ (("bicodec", True), (None, True), "invalid_value"),
+ ((None, True), ("bicodec", True), "model_downloading"),
+ (("bicodec", True), (None, False), "model_downloading"),
+ (("csm", True), None, "invalid_value"),
+ ((None, False), None, "model_lookup_failed"),
+ ],
+)
+def test_speech_download_admission(hub, monkeypatch, sidecar, probe, code):
+ metadata, weights = [], []
+ monkeypatch.setattr(
+ "utils.models.model_config.detect_audio_type_checked",
+ lambda repo, **kw: metadata.append(kw) or sidecar,
+ )
+ monkeypatch.setattr(
+ auto_dl, "_probe_remote_gguf_audio_type", lambda *a: weights.append(a) or probe
+ )
+ result = asyncio.run(auto_dl.maybe_auto_download("unsloth/tts-GGUF:Q8_0", require_speech = True))
+ assert result.code == code
+ assert result.status == (400 if code == "invalid_value" else 503)
+ assert bool(hub["started"]) is (code == "model_downloading")
+ assert metadata == [{"hf_token": False, "revision": hub["info"].sha}]
+ assert weights == (
+ []
+ if probe is None
+ else [("unsloth/tts-GGUF", "model-Q8_0-00001-of-00002.gguf", None, hub["info"].sha)]
+ )
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index f35d0568d82..8878e42706a 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -106,12 +106,18 @@ class _LoadRecorder:
current_subject = None,
*,
current_request_counted = False,
+ cache_environment = None,
+ anonymous_hf_access = False,
+ speech_codec_path = None,
):
# Mirror the production load boundary before recording any replacement.
await inference_route._wait_for_model_switch_idle(
current_request_counted = current_request_counted
)
self.calls.append(request)
+ self.cache_environment = cache_environment
+ self.anonymous_hf_access = anonymous_hf_access
+ self.speech_codec_path = speech_codec_path
if self.fail:
from fastapi import HTTPException
raise HTTPException(status_code = 503, detail = "load failed")
@@ -140,6 +146,7 @@ def _wire(monkeypatch, *, enabled, resolves_to, backend, recorder):
# gate that auto-switch already owns, so it calls the impl directly).
monkeypatch.setattr(inference_route, "_load_model_impl", recorder)
monkeypatch.setattr(inference_route, "_auto_switch_waiters", {})
+ monkeypatch.setattr(inference_route, "_preflight_speech_codec_for_switch", lambda *_a: None)
async def _noop_reject(*_args, **_kwargs):
@@ -5229,39 +5236,35 @@ def test_a_count_never_spawns_mcp_servers():
assert "get_enabled_mcp_tools" not in called
-def test_audio_generate_is_reload_only(monkeypatch):
- # Codex P2: /audio/generate must not switch to a client-named GGUF. A local
- # GGUF's audio-input capability is not a cheap pre-load probe (the mmproj signal
- # can't tell an audio projector from a vision one), so resolving the client model
- # could evict the working audio model for a target that then fails the audio
- # check. Only the idle-stash restore runs: the hook gets the reload-only sentinel.
- from models.inference import ChatCompletionRequest
-
+def _capture_audio_switch(monkeypatch):
class _Reached(Exception):
pass
captured = {}
- async def _capture(
- model,
- request,
- subject,
- *,
- require_vision = False,
- claim_resident = True,
- ):
+ async def _capture(model, request, subject, **kwargs):
captured["model"] = model
- captured["claim_resident"] = claim_resident
+ captured.update(kwargs)
raise _Reached()
monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _capture)
+ return _Reached, captured
+
+
[email protected]("model", [None, "", "org/B-GGUF"])
+def test_audio_generate_model_selection(monkeypatch, model):
+ from models.inference import ChatCompletionRequest
+
+ reached, captured = _capture_audio_switch(monkeypatch)
payload = ChatCompletionRequest(
- model = "org/B-GGUF", messages = [{"role": "user", "content": "say hi"}]
+ messages = [{"role": "user", "content": "say hi"}],
+ **({"model": model} if model is not None else {}),
)
- with pytest.raises(_Reached):
+ with pytest.raises(reached):
asyncio.run(inference_route.generate_audio(payload, object(), "tester"))
- assert captured["model"] == inference_route._RELOAD_ONLY_MODEL
+ assert captured["model"] == (model or inference_route._RELOAD_ONLY_MODEL)
assert captured["claim_resident"] is False
+ assert captured["require_speech"] is True
def test_note_model_unloaded_clears_reload_stash(monkeypatch):
@@ -9065,6 +9068,74 @@ def _local_checkpoint(root, name = "Qwen3-MLX-4bit"):
return path
+def _complete_minimax_pipeline(root):
+ pipeline = root / "MiniMax-Music3"
+ pipeline.mkdir()
+ component_specs = {
+ "condition_encoder": ("diffusers", "MiniMaxMusic3ConditionEncoder"),
+ "language_model": ("transformers", "Qwen3ForCausalLM"),
+ "rvq_depth_decoder": ("diffusers", "MiniMaxMusic3RVQDepthDecoder"),
+ "scheduler": ("diffusers", "FlowMatchEulerDiscreteScheduler"),
+ "tokenizer": ("transformers", "Qwen2Tokenizer"),
+ "transformer": ("diffusers", "MiniMaxMusic3Transformer1DModel"),
+ "vocoder": ("diffusers", "MiniMaxMusic3Vocoder"),
+ }
+ index = {
+ "_class_name": "MiniMaxMusic3ModularPipeline",
+ "_blocks_class_name": "MiniMaxMusic3Blocks",
+ **{
+ component: [*component_specs[component], {"subfolder": component}]
+ for component in component_specs
+ },
+ }
+ (pipeline / "modular_model_index.json").write_text(json.dumps(index))
+ for component in component_specs:
+ directory = pipeline / component
+ directory.mkdir()
+ if component == "scheduler":
+ (directory / "scheduler_config.json").write_text(
+ json.dumps({"_class_name": "Scheduler"})
+ )
+ elif component == "tokenizer":
+ (directory / "tokenizer_config.json").write_text(
+ json.dumps({"tokenizer_class": "PreTrainedTokenizerFast"})
+ )
+ (directory / "tokenizer.json").write_text(
+ json.dumps({"model": {"type": "BPE", "vocab": {}, "merges": []}})
+ )
+ else:
+ (directory / "config.json").write_text(json.dumps({"_class_name": "Component"}))
+ stem = "model" if component == "language_model" else "diffusion_pytorch_model"
+ (directory / f"{stem}.safetensors").write_bytes(_safetensors_bytes())
+ return pipeline
+
+
+ "repo", ["unsloth/Spark-TTS-0.5B", "SparkAudio/Spark-TTS-0.5B", "org/custom-voice"]
+)
+def test_a_downloaded_spark_repository_resolves_to_its_llm_checkpoint(tmp_path, repo):
+ snapshot = tmp_path / "models--unsloth--Spark-TTS-0.5B" / "snapshots" / "revision"
+ llm = snapshot / "LLM"
+ llm.mkdir(parents = True)
+ (llm / "config.json").write_text(
+ json.dumps({"architectures": ["Qwen2ForCausalLM"], "model_type": "qwen2"})
+ )
+ (llm / "model.safetensors").write_bytes(_safetensors_bytes())
+ tokens = {
+ str(i): {"content": f"<|bicodec_{name}_0|>"}
+ for i, name in enumerate(("semantic", "global"))
+ }
+ (llm / "tokenizer_config.json").write_text(json.dumps({"added_tokens_decoder": tokens}))
+ info = types.SimpleNamespace(id = repo, model_id = repo, path = str(snapshot), partial = False)
+ entry = resolver._local_weights_entry(repo, info)
+ assert entry is not None
+ assert entry.load_path == str(llm)
+ assert entry.is_gguf is False
+ assert inference_route._target_speech_audio_type(entry.load_path, False) == "bicodec"
+ (llm / "tokenizer_config.json").write_text("{}")
+ assert resolver._local_weights_entry("org/text", info) is None
+
+
def test_a_diffusers_pipeline_is_not_a_servable_chat_model(tmp_path):
# The Images and Video backends own these; /v1/chat/completions cannot serve them.
from types import SimpleNamespace
@@ -10453,3 +10524,183 @@ def test_a_serving_hf_cache_row_is_reported_loaded(tmp_path, monkeypatch):
assert [(is_gguf, quants, resident) for _i, is_gguf, quants, resident in rows] == [
(False, (), True)
]
+
+
+def _speech_case(
+ monkeypatch,
+ audio_type = "snac",
+ gguf = True,
+):
+ backend = _FakeBackend("org/A-GGUF")
+ recorder = _LoadRecorder(backend)
+ target = ("/local/B.gguf", "Q8_0", "org/B-GGUF") if gguf else ("/local/B", None, "org/B-GGUF")
+ _wire(monkeypatch, enabled = True, resolves_to = target, backend = backend, recorder = recorder)
+ monkeypatch.setattr(inference_route, "_target_speech_audio_type", lambda *_a: audio_type)
+ return backend, recorder
+
+
+ "audio_type,context,text,instructions,status",
+ [
+ (None, 8192, "hi", "", 400),
+ ("snac", 8192, "hi", "", None),
+ ("snac", 512, "x" * 1000, "", 400),
+ ("snac", None, "hi", "", None),
+ ("higgs_tts2", 512, "hi", "x" * 1000, 400),
+ ("minimax_music3", None, "hi", "", 400),
+ ("minimax_music3", None, "hi", "a warm ballad", None),
+ ],
+)
+def test_speech_switch_admission(monkeypatch, audio_type, context, text, instructions, status):
+ backend, recorder = _speech_case(monkeypatch, audio_type, audio_type in (None, "snac"))
+ monkeypatch.setattr(inference_route, "_target_effective_context_length", lambda *_a: context)
+ call = inference_route._maybe_auto_switch_model(
+ "org/B-GGUF",
+ object(),
+ "tester",
+ require_speech = True,
+ speech_budget = {"text": text, "instructions": instructions},
+ )
+ if status:
+ with pytest.raises(HTTPException) as error:
+ asyncio.run(call)
+ assert error.value.status_code == status
+ assert recorder.calls == [] and backend.model_identifier == "org/A-GGUF"
+ else:
+ asyncio.run(call)
+ assert len(recorder.calls) == 1
+
+
[email protected]("audio_type", ["snac", "bicodec", "dac", "higgs_tts2"])
[email protected]("token", [None, "caller-token"])
+def test_speech_switch_preserves_staged_assets_and_identity(monkeypatch, audio_type, token):
+ from starlette.requests import Request
+
+ _, recorder = _speech_case(monkeypatch, audio_type, audio_type != "higgs_tts2")
+ staged = inference_route._SpeechCodecPreflightResult(
+ {"HF_HUB_CACHE": "/captured/hub"}, "/captured/codec"
+ )
+ seen = []
+ monkeypatch.setattr(
+ inference_route, "_preflight_speech_codec_for_switch", lambda *a: seen.append(a) or staged
+ )
+ headers = [(b"x-unsloth-hf-token", token.encode())] if token else []
+ request = Request({"type": "http", "path": "/v1/audio/speech", "headers": headers})
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "org/B-GGUF", request, "tester", require_speech = True
+ )
+ )
+ assert seen[0][-1] == token
+ assert recorder.cache_environment == staged.cache_environment
+ assert recorder.speech_codec_path == staged.codec_path
+ assert recorder.anonymous_hf_access is (token is None)
+ assert recorder.calls[0].hf_token == token
+
+
[email protected]("condition", ["complete", "missing", "unsafe"])
[email protected]("token", [None, "caller-token"])
+def test_higgs_companion_preflight_before_eviction(tmp_path, monkeypatch, condition, token):
+ import huggingface_hub
+ from core.inference import native_audio
+ from utils import hf_cache_settings, security, utils
+ from starlette.requests import Request
+
+ preflight = inference_route._preflight_speech_codec_for_switch
+ backend, recorder = _speech_case(monkeypatch, "higgs_tts2", False)
+ monkeypatch.setattr(inference_route, "_preflight_speech_codec_for_switch", preflight)
+ codec = tmp_path / "codec"
+ codec.mkdir()
+ (codec / "config.json").write_text(json.dumps({"model_type": "higgs_audio_v2_tokenizer"}))
+ if condition != "missing":
+ (codec / "model.safetensors").write_bytes(_safetensors_bytes())
+ cache = hf_cache_settings.HuggingFaceCachePaths(
+ tmp_path, tmp_path / "hub", tmp_path / "xet", "studio"
+ )
+ seen = []
+ monkeypatch.setattr(hf_cache_settings, "get_hf_cache_paths", lambda: cache)
+ monkeypatch.setattr(utils, "hf_env_offline", lambda: True)
+ monkeypatch.setattr(
+ native_audio, "native_audio_security_targets", lambda p, *_a: [p, "org/codec"]
+ )
+ monkeypatch.setattr(
+ huggingface_hub, "snapshot_download", lambda repo, **kw: seen.append(kw) or str(codec)
+ )
+ monkeypatch.setattr(
+ security,
+ "evaluate_file_security",
+ lambda *_a, **_k: types.SimpleNamespace(blocked = condition == "unsafe", reason = "unsafe"),
+ )
+ headers = [(b"x-unsloth-hf-token", token.encode())] if token else []
+ request = Request({"type": "http", "path": "/v1/audio/speech", "headers": headers})
+ call = inference_route._maybe_auto_switch_model(
+ "org/B-GGUF", request, "tester", require_speech = True
+ )
+ if condition == "complete":
+ asyncio.run(call)
+ assert len(recorder.calls) == 1
+ else:
+ with pytest.raises(HTTPException) as error:
+ asyncio.run(call)
+ assert error.value.status_code == 503
+ assert recorder.calls == [] and backend.is_loaded
+ assert seen == [
+ {"token": token or False, "cache_dir": str(cache.hub_cache), "local_files_only": True}
+ ]
+
+
+def test_speech_budget_rechecks_changing_load_override(monkeypatch):
+ backend, recorder = _speech_case(monkeypatch)
+ reads = []
+
+ def override(*_a):
+ value = 8192 if not reads else 512
+ reads.append(value)
+ return "org/B-GGUF:Q8_0", {"max_seq_length": value}
+
+ monkeypatch.setattr(settings, "resolve_override_for_load", override)
+ with pytest.raises(HTTPException) as error:
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "org/B-GGUF",
+ object(),
+ "tester",
+ require_speech = True,
+ speech_budget = {"text": "x" * 1000},
+ )
+ )
+ assert reads == [8192, 512] and error.value.status_code == 400
+ assert recorder.calls == [] and backend.model_identifier == "org/A-GGUF"
+
+
[email protected]("fault", [None, "weights", "metadata", "remote-code"])
+def test_minimax_pipeline_completeness(tmp_path, monkeypatch, fault):
+ pipeline = _complete_minimax_pipeline(tmp_path)
+ monkeypatch.setattr(resolver, "_host_can_serve_minimax_music3", lambda: True)
+ if fault == "weights":
+ (pipeline / "vocoder" / "diffusion_pytorch_model.safetensors").unlink()
+ elif fault == "metadata":
+ (pipeline / "tokenizer" / "tokenizer.json").write_text("not-json")
+ elif fault == "remote-code":
+ (pipeline / "language_model" / "config.json").write_text(
+ json.dumps({"auto_map": {"AutoModel": "modeling.Custom"}})
+ )
+ info = types.SimpleNamespace(id = "MiniMaxAI/MiniMax-Music3", path = str(pipeline))
+ assert resolver.local_servable_model(info) == ((False, ()) if fault is None else None)
+
+
+ "audio_type,allowed",
+ [
+ ("moss_tts_local", False),
+ ("moss_tts_nano", False),
+ ("higgs_tts3", False),
+ ("higgs_tts2", True),
+ ],
+)
+def test_speech_probe_refuses_remote_code(monkeypatch, audio_type, allowed):
+ from utils.models import model_config
+ monkeypatch.setattr(model_config, "detect_audio_type", lambda *_a, **_kw: audio_type)
+ assert inference_route._target_speech_audio_type("/local/model", False) == (
+ audio_type if allowed else None
+ )
diff --git a/studio/backend/tests/test_third_party_source.py b/studio/backend/tests/test_third_party_source.py
index 87c915524ce..d7f3d53ca8e 100644
--- a/studio/backend/tests/test_third_party_source.py
+++ b/studio/backend/tests/test_third_party_source.py
@@ -671,6 +671,7 @@ def test_dac_weights_use_immutable_revision_and_active_cache(monkeypatch, tmp_pa
"repo_id": source._DAC_REPOSITORY,
"filename": source._DAC_FILENAME,
"revision": source._DAC_REVISION,
+ "token": None,
"cache_dir": str(hub_cache),
"local_files_only": False,
}
diff --git a/studio/backend/utils/audio_tokens.py b/studio/backend/utils/audio_tokens.py
index 9046292ef1e..d66ea1442d9 100644
--- a/studio/backend/utils/audio_tokens.py
+++ b/studio/backend/utils/audio_tokens.py
@@ -35,6 +35,7 @@ VALID_AUDIO_TYPES = (
# Emit speech; a chat turn sent to one comes back as audio, never as text.
TTS_AUDIO_TYPES = frozenset({"snac", "csm", "bicodec", "dac"})
+GGUF_TTS_AUDIO_TYPES = frozenset({"snac", "bicodec", "dac"})
OUTPUT_AUDIO_TYPES = TTS_AUDIO_TYPES | NATIVE_OUTPUT_AUDIO_TYPES
@@ -108,6 +109,41 @@ def classify_audio_tokens(tok_config: dict) -> Optional[str]:
return None
+# Keep token order and strictness aligned with LlamaCppBackend._detect_audio_type_strict.
+SNAC_PROBE_TOKEN_IDS = (128258, 128259)
+
+GGUF_AUDIO_CLASSIFIER_TOKENS = frozenset(
+ {
+ "<|AUDIO|>",
+ "<|audio_eos|>",
+ "<|startoftranscript|>",
+ "<audio_soft_token>",
+ "<|audio|>",
+ "<|bicodec_semantic_0|>",
+ "<|bicodec_global_0|>",
+ "<|c1_0|>",
+ "<|c2_0|>",
+ }
+)
+
+
+def classify_gguf_vocab_audio_type(tokens: set, snac_probe_is_codes: bool) -> Optional[str]:
+ if snac_probe_is_codes:
+ return "snac"
+ if "<|AUDIO|>" in tokens and "<|audio_eos|>" in tokens:
+ return "csm"
+ if "<|startoftranscript|>" in tokens:
+ return "whisper"
+ # Match the serving detector: audio-input markers take precedence over codecs.
+ if "<audio_soft_token>" in tokens or "<|audio|>" in tokens:
+ return "audio_vlm"
+ if "<|bicodec_semantic_0|>" in tokens and "<|bicodec_global_0|>" in tokens:
+ return "bicodec"
+ if "<|c1_0|>" in tokens and "<|c2_0|>" in tokens:
+ return "dac"
+ return None
+
+
def is_audio_input_type(audio_type: Optional[str]) -> bool:
"""True if an audio_type accepts audio input: whisper (ASR), audio_vlm (Gemma3n)."""
return audio_type in ("whisper", "audio_vlm")
diff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.py
index 5d133cbeb30..f479b3adc92 100644
--- a/studio/backend/utils/models/gguf_metadata.py
+++ b/studio/backend/utils/models/gguf_metadata.py
@@ -3,7 +3,7 @@
"""``general.*`` reader for GGUF headers, used by ``detect_mmproj_file`` to
pair weights and projectors via ``general.base_model.0.repo_url``. ~30 ms
-per file, cached by (path, mtime, size)."""
+per file, cached by resolved path and platform file identity."""
from __future__ import annotations
@@ -11,7 +11,7 @@ import os
import struct
import threading
from pathlib import Path
-from typing import Dict, Optional, Tuple
+from typing import BinaryIO, Dict, Optional, Tuple
from loggers import get_logger
@@ -41,7 +41,7 @@ _WANTED_GENERAL_KEYS: frozenset[str] = frozenset(
# Cache failed parses too so a broken file is not retried each scan.
-_CacheKey = Tuple[str, int, int]
+_CacheKey = Tuple[str, int, int, int, int, int]
_METADATA_CACHE: Dict[_CacheKey, Optional[Dict[str, str]]] = {}
_CACHE_LOCK = threading.Lock()
_CACHE_MAX_ENTRIES = 4096
@@ -52,6 +52,8 @@ _BOOL_CACHE: Dict[Tuple[_CacheKey, str], Optional[bool]] = {}
_STRING_CACHE: Dict[Tuple[_CacheKey, str], Optional[str]] = {}
+_TTS_AUDIO_TYPE_CACHE: Dict[_CacheKey, Optional[str]] = {}
+
# Whether the GGUF tensor table contains a sequence-classification head. None
# means the file could not be read or parsed, so callers can fail closed.
_CLASSIFIER_HEAD_CACHE: Dict[_CacheKey, Optional[bool]] = {}
@@ -76,7 +78,14 @@ def _cache_key(path: str) -> Optional[_CacheKey]:
resolved = str(Path(path).resolve())
except OSError:
resolved = str(path)
- return (resolved, st.st_mtime_ns, st.st_size)
+ return (
+ resolved,
+ st.st_mtime_ns,
+ st.st_size,
+ int(getattr(st, "st_ctime_ns", 0)),
+ int(getattr(st, "st_dev", 0)),
+ int(getattr(st, "st_ino", 0)),
+ )
def read_gguf_general_metadata(path: str) -> Optional[Dict[str, str]]:
@@ -611,6 +620,135 @@ def _read_gguf_string(path: str, wanted_key: str) -> Optional[str]:
return result
+_MAX_GGUF_VOCAB_ENTRIES = 2_000_000
+
+
+def _parse_gguf_marker_tokens_stream(f: BinaryIO) -> Optional[Tuple[list[str], bool]]:
+ from utils.audio_tokens import GGUF_AUDIO_CLASSIFIER_TOKENS, SNAC_PROBE_TOKEN_IDS
+
+ marker_bytes = {token.encode("utf-8"): token for token in GGUF_AUDIO_CLASSIFIER_TOKENS}
+
+ try:
+ head = f.read(24)
+ if len(head) < 24:
+ return None
+ magic, _version, _tcount, kv_count = struct.unpack("<IIQQ", head)
+ if magic != _GGUF_MAGIC:
+ return None
+ marker_tokens: dict[str, int] = {}
+ token_types: Optional[bytes] = None
+ snac_probe: Optional[dict[int, bool]] = None
+ for _ in range(kv_count):
+ klen_bytes = f.read(8)
+ if len(klen_bytes) < 8:
+ return None
+ klen = struct.unpack("<Q", klen_bytes)[0]
+ if klen > 1 << 20:
+ return None
+ key = f.read(klen).decode("utf-8", "replace")
+ vt_bytes = f.read(4)
+ if len(vt_bytes) < 4:
+ return None
+ vtype = struct.unpack("<I", vt_bytes)[0]
+ if key == "tokenizer.ggml.token_type" and vtype == 9:
+ raw_header = f.read(12)
+ if len(raw_header) != 12:
+ return None
+ atype, alen = struct.unpack("<IQ", raw_header)
+ if atype != 5 or alen > _MAX_GGUF_VOCAB_ENTRIES:
+ return None
+ raw_types = f.read(4 * alen)
+ if len(raw_types) != 4 * alen:
+ return None
+ token_types = raw_types
+ continue
+ if key != "tokenizer.ggml.tokens" or vtype != 9:
+ if not _skip_gguf_value(f, vtype):
+ return None
+ continue
+ raw_header = f.read(12)
+ if len(raw_header) != 12:
+ return None
+ atype, alen = struct.unpack("<IQ", raw_header)
+ if atype != 8 or alen > _MAX_GGUF_VOCAB_ENTRIES:
+ return None
+ # SNAC classification depends on the tokens at these exact IDs.
+ snac_probe = dict.fromkeys(SNAC_PROBE_TOKEN_IDS, False)
+ for index in range(alen):
+ raw_length = f.read(8)
+ if len(raw_length) != 8:
+ return None
+ slen = struct.unpack("<Q", raw_length)[0]
+ if slen > 1 << 20:
+ return None
+ raw = f.read(slen)
+ if len(raw) != slen:
+ return None
+ if index in snac_probe:
+ snac_probe[index] = b"<custom_token_" in raw
+ marker = marker_bytes.get(raw)
+ if marker is not None:
+ if marker in marker_tokens:
+ return None
+ marker_tokens[marker] = index
+ if snac_probe is None:
+ return None
+ # llama.cpp does not parse NORMAL vocabulary entries as special markers.
+ markers = [
+ token
+ for token, index in marker_tokens.items()
+ if token_types is not None
+ and 4 * (index + 1) <= len(token_types)
+ and struct.unpack_from("<i", token_types, 4 * index)[0] in {2, 3, 4}
+ ]
+ return markers, all(snac_probe.values())
+ except (OSError, struct.error) as e:
+ logger.debug(f"_parse_gguf_marker_tokens_stream: cannot read stream: {e}")
+ return None
+
+
+def _parse_gguf_marker_tokens(path: str) -> Optional[Tuple[list[str], bool]]:
+ try:
+ with open(path, "rb") as f:
+ return _parse_gguf_marker_tokens_stream(f)
+ except OSError as e:
+ logger.debug(f"_parse_gguf_marker_tokens: cannot read {path}: {e}")
+ return None
+
+
+def classify_gguf_tts_audio_prefix(data: bytes) -> Tuple[Optional[str], bool]:
+ from io import BytesIO
+ from utils.audio_tokens import classify_gguf_vocab_audio_type, is_tts_audio_type
+
+ parsed = _parse_gguf_marker_tokens_stream(BytesIO(data))
+ if parsed is None:
+ return None, False
+ audio_type = classify_gguf_vocab_audio_type(set(parsed[0]), parsed[1])
+ return (audio_type if is_tts_audio_type(audio_type) else None), True
+
+
+def read_gguf_tts_audio_type(path: str) -> Optional[str]:
+ from utils.audio_tokens import classify_gguf_vocab_audio_type, is_tts_audio_type
+
+ fkey = _cache_key(path)
+ if fkey is None:
+ return None
+ with _CACHE_LOCK:
+ if fkey in _TTS_AUDIO_TYPE_CACHE:
+ return _TTS_AUDIO_TYPE_CACHE[fkey]
+ parsed = _parse_gguf_marker_tokens(path)
+ audio_type = classify_gguf_vocab_audio_type(set(parsed[0]), parsed[1]) if parsed else None
+ result = audio_type if is_tts_audio_type(audio_type) else None
+ with _CACHE_LOCK:
+ while len(_TTS_AUDIO_TYPE_CACHE) >= _CACHE_MAX_ENTRIES:
+ try:
+ _TTS_AUDIO_TYPE_CACHE.pop(next(iter(_TTS_AUDIO_TYPE_CACHE)))
+ except StopIteration:
+ break
+ _TTS_AUDIO_TYPE_CACHE[fkey] = result
+ return result
+
+
def read_gguf_chat_template(path: str) -> Optional[str]:
template = _read_gguf_string(path, "tokenizer.chat_template")
if isinstance(template, str) and template.strip():
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 7f8a69cbd2c..2bbef0d87f4 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -49,7 +49,7 @@ import re
import subprocess
import sys
from pathlib import Path
-from typing import Callable, List, Tuple, Union
+from typing import Callable, List, Tuple
import hashlib
import json
import threading
@@ -1119,7 +1119,7 @@ def _hub_model_info(
# Revision-less entries keep the historical 3-part key; pinned entries append revision.
-_CapabilityCacheKey = Union[Tuple[str, Optional[str], bool], Tuple[str, Optional[str], bool, str]]
+_CapabilityCacheKey = Tuple[Any, ...]
_vision_detection_cache: Dict[_CapabilityCacheKey, bool] = {}
_vision_cache_lock = threading.Lock()
@@ -1320,6 +1320,36 @@ _AUDIO_OFFLINE_MISS_TTL_S = 60.0
_audio_offline_miss_cache: Dict[_CapabilityCacheKey, float] = {}
+def _local_audio_metadata_fingerprint(model_name: str) -> tuple:
+ try:
+ root = Path(normalize_path(model_name)).expanduser()
+ if root.is_file():
+ root = root.parent
+ identities = []
+ for relative in (
+ "config.json",
+ "modular_model_index.json",
+ *_AUDIO_TOKENIZER_CONFIG_PATHS,
+ ):
+ candidate = root / relative
+ try:
+ stat = candidate.stat()
+ identity = (
+ stat.st_mtime_ns,
+ stat.st_size,
+ getattr(stat, "st_ctime_ns", None),
+ getattr(stat, "st_dev", None),
+ getattr(stat, "st_ino", None),
+ )
+ except OSError:
+ identity = None
+ identities.append((relative, identity))
+ return tuple(identities)
+ except Exception:
+ # Keep cache-key construction from turning an unreadable path into a request failure.
+ return (("unreadable", None),)
+
+
def detect_audio_type(
model_name: str,
hf_token: Optional[str] = None,
@@ -1379,6 +1409,9 @@ def detect_audio_type_checked(
# does not put an anonymous caller back on the wire. Inconclusive for it instead.
if effective_offline and is_anonymous(hf_token) and not is_local_path(model_name):
return None, False
+ local_fingerprint = (
+ _local_audio_metadata_fingerprint(model_name) if is_local_path(model_name) else None
+ )
# Checked on the RAW name, before the casing resolution below, because resolving a
# repo id that is not in the cache walks every cache directory, and that walk is the
# cost this cache exists to avoid. A casing variant just takes its own entry, which
@@ -1390,6 +1423,8 @@ def detect_audio_type_checked(
)
if revision is not None:
miss_key += (revision,)
+ if local_fingerprint is not None:
+ miss_key += (local_fingerprint,)
if effective_offline:
seen_at = _audio_offline_miss_cache.get(miss_key)
if seen_at is not None and time.monotonic() - seen_at < _AUDIO_OFFLINE_MISS_TTL_S:
@@ -1410,6 +1445,8 @@ def detect_audio_type_checked(
)
if revision is not None:
cache_key += (revision,)
+ if local_fingerprint is not None:
+ cache_key += (local_fingerprint,)
if cache_key in _audio_detection_cache:
# Only definitive results are cached, so a hit is definitive by construction.
return _audio_detection_cache[cache_key], True
diff --git a/studio/backend/utils/third_party_source.py b/studio/backend/utils/third_party_source.py
index ed6b92e1b8b..dd033d2022b 100644
--- a/studio/backend/utils/third_party_source.py
+++ b/studio/backend/utils/third_party_source.py
@@ -27,6 +27,7 @@ from types import ModuleType
from filelock import FileLock, Timeout
+from hub.utils.hf_tokens import HfTokenArg
from utils.native_path_leases import child_env_without_native_path_secret
from utils.paths.storage_roots import cache_root
from utils.subprocess_compat import (
@@ -877,12 +878,17 @@ def _default_legacy_dac_weights_path() -> Path | None:
return Path.home() / ".cache" / "outeai" / "dac" / _DAC_FILENAME
-def ensure_dac_speech_weights(legacy_path: Path | str | None = None) -> Path:
+def ensure_dac_speech_weights(
+ legacy_path: Path | str | None = None,
+ *,
+ hub_cache: Path | str | None = None,
+ hf_token: HfTokenArg = None,
+) -> Path:
from huggingface_hub import hf_hub_download
from utils.hf_cache_settings import active_hf_hub_cache
from utils.utils import hf_env_offline
- hub_cache = Path(active_hf_hub_cache())
+ hub_cache = Path(hub_cache) if hub_cache is not None else Path(active_hf_hub_cache())
destination = (
hub_cache
/ "studio-pinned-artifacts"
@@ -951,6 +957,7 @@ def ensure_dac_speech_weights(legacy_path: Path | str | None = None) -> Path:
repo_id = _DAC_REPOSITORY,
filename = _DAC_FILENAME,
revision = _DAC_REVISION,
+ token = hf_token,
cache_dir = str(hub_cache),
local_files_only = offline,
)