Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Tests for tokenizer-based audio_type detection, covering Gemma 3n
(<audio_soft_token>) and Gemma 4 (<|audio|>) audio-input tokens."""
from __future__ import annotations
import json
import os
import pytest
from utils.audio_tokens import AUDIO_TOKEN_PATTERNS
from utils.models.model_config import (
detect_audio_type_checked,
is_audio_input_type,
)
def test_curated_native_audio_repos_are_detected_without_hub_reads():
expected = {
"bosonai/higgs-tts-2-3b-base": "higgs_tts2",
"OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5": "moss_tts_local",
"OpenMOSS-Team/MOSS-TTS-Nano-100M": "moss_tts_nano",
"multimodalart/higgs-audio-v3-tts-4b-transformers": "higgs_tts3",
"MiniMaxAI/MiniMax-Music3": "minimax_music3",
}
for repo, audio_type in expected.items():
assert detect_audio_type_checked(repo) == (audio_type, True)
def test_local_native_audio_model_type_is_detected(tmp_path):
(tmp_path / "config.json").write_text(
json.dumps({"model_type": "moss_tts_nano"}),
encoding = "utf-8",
)
assert detect_audio_type_checked(str(tmp_path)) == ("moss_tts_nano", True)
def test_replacing_local_audio_metadata_invalidates_the_cached_verdict(tmp_path, monkeypatch):
from utils.models import model_config
monkeypatch.setattr(model_config, "_audio_detection_cache", {})
config = tmp_path / "config.json"
replacement = tmp_path / "replacement.json"
config.write_text('{"model_type":"moss_tts_nano"}', encoding = "utf-8")
replacement.write_text('{"model_type":"not_audio_xyz"}', encoding = "utf-8")
assert config.stat().st_size == replacement.stat().st_size
assert model_config.detect_audio_type_checked(str(tmp_path))[0] == "moss_tts_nano"
original = config.stat()
os.replace(replacement, config)
os.utime(config, ns = (config.stat().st_atime_ns, original.st_mtime_ns))
assert config.stat().st_size == original.st_size
assert config.stat().st_mtime_ns == original.st_mtime_ns
assert model_config.detect_audio_type_checked(str(tmp_path))[0] is None
def _classify(tokens: list[str]) -> str | None:
"""Mirror _check_token_patterns: first match in dict order wins."""
for audio_type, check in AUDIO_TOKEN_PATTERNS.items():
if check(tokens):
return audio_type
return None
def test_gemma3n_audio_soft_token_is_audio_vlm():
assert _classify(["<bos>", "<audio_soft_token>", "<image_soft_token>"]) == "audio_vlm"
def test_gemma4_pipe_audio_token_is_audio_vlm():
assert _classify(["<bos>", "<|image|>", "<|audio|>"]) == "audio_vlm"
def test_csm_uppercase_audio_not_classified_as_audio_vlm():
tokens = ["<|AUDIO|>", "<|audio_eos|>"]
assert _classify(tokens) == "csm"
def test_audio_vlm_and_whisper_accept_audio_input():
assert is_audio_input_type("audio_vlm") is True
assert is_audio_input_type("whisper") is True
assert is_audio_input_type("snac") is False
assert is_audio_input_type(None) is False
def test_non_audio_tokens_classify_none():
assert _classify(["<bos>", "<eos>", "<pad>"]) is None
def test_orpheus_snac_codebook_beats_a_stray_audio_marker():
"""SNAC codebook tokens must outrank a lone <|audio|> marker, or TTS is misread as audio input."""
tokens = ["<|audio|>"] + [f"<custom_token_{i}>" for i in range(28683)]
assert _classify(tokens) == "snac"
assert is_audio_input_type(_classify(tokens)) is False
def test_a_codec_family_is_not_shadowed_by_a_stray_audio_marker():
"""The same precedence has to hold for every output codec, not just snac."""
assert _classify(["<|audio|>", "<|bicodec_semantic_0|>"]) == "bicodec"
assert (
_classify(
[
"<|audio|>",
"<|audio_start|>",
"<|audio_end|>",
"<|text_start|>",
"<|text_end|>",
]
)
== "dac"
)
class _Resp:
def __init__(
self,
status_code: int,
payload = None,
):
self.status_code = status_code
self.ok = 200 <= status_code < 300
self._payload = payload
def json(self):
if self._payload is None:
raise ValueError("no body")
return self._payload
def _detect_checked(
monkeypatch,
responses,
model = "acme/tts-model",
):
"""Drive detect_audio_type_checked with a faked Hub, no local cache."""
from utils.models import model_config as mc
monkeypatch.setattr(mc, "_audio_detection_cache", {})
monkeypatch.setattr(mc, "get_cache_path", lambda *a, **k: None)
monkeypatch.setattr(mc, "_env_offline", lambda: False)
import requests
monkeypatch.setattr(requests, "get", lambda url, **kw: responses.pop(0))
return mc.detect_audio_type_checked(model)
def test_a_gated_repo_is_not_reported_as_definitively_non_audio(monkeypatch):
audio_type, definitive = _detect_checked(monkeypatch, [_Resp(401), _Resp(401)])
assert audio_type is None
assert definitive is False
def test_a_readable_repo_without_audio_tokens_is_definitive(monkeypatch):
plain = {"added_tokens_decoder": {"0": {"content": "<bos>"}}}
audio_type, definitive = _detect_checked(monkeypatch, [_Resp(200, plain), _Resp(404)])
assert audio_type is None
assert definitive is True
def test_a_detected_codec_is_definitive(monkeypatch):
snac = {
"added_tokens_decoder": {str(i): {"content": f"<custom_token_{i}>"} for i in range(10_001)}
}
audio_type, definitive = _detect_checked(monkeypatch, [_Resp(200, snac)])
assert audio_type == "snac"
assert definitive is True
def test_a_local_path_never_reaches_the_hub(monkeypatch, tmp_path):
"""Local paths must never reach the Hub fetch, which would stall each adapter directory on timeouts."""
from utils.models import model_config
# Recorded, not raised: the fetch loop swallows exceptions as transient failures.
fetched = []
import requests
monkeypatch.setattr(requests, "get", lambda url, **kwargs: fetched.append(url))
adapter = tmp_path / "adapter"
adapter.mkdir()
(adapter / "adapter_config.json").write_text("{}", encoding = "utf-8")
result, definitive = model_config._detect_audio_from_tokenizer(str(adapter))
assert fetched == [], fetched
assert result is None
assert definitive is False
def test_an_offline_miss_is_not_reprobed_on_every_poll(monkeypatch, tmp_path):
"""/loras probes each checkpoint and its base, and neither answers offline; a non-definitive
result is never cached, so the walk repeated on every poll on the event loop."""
from utils.models import model_config
monkeypatch.setattr(model_config, "_audio_detection_cache", {})
monkeypatch.setattr(model_config, "_audio_offline_miss_cache", {})
probes = []
monkeypatch.setattr(
model_config,
"_detect_audio_from_tokenizer",
lambda name, token = None, **kw: (probes.append(name), (None, False))[1],
)
for _ in range(5):
assert model_config.detect_audio_type_checked(
"org/not-downloaded", local_files_only = True
) == (None, False)
assert probes == ["org/not-downloaded"], probes
def test_the_offline_miss_expires_so_a_later_download_is_seen(monkeypatch):
"""Bounded, not permanent: a later download or finished training run can add the tokenizer."""
from utils.models import model_config
monkeypatch.setattr(model_config, "_audio_detection_cache", {})
monkeypatch.setattr(model_config, "_audio_offline_miss_cache", {})
answers = iter([(None, False), ("snac", True)])
monkeypatch.setattr(
model_config,
"_detect_audio_from_tokenizer",
lambda name, token = None, **kw: next(answers),
)
clock = [1000.0]
monkeypatch.setattr(model_config.time, "monotonic", lambda: clock[0])
assert model_config.detect_audio_type_checked("org/m", local_files_only = True)[0] is None
clock[0] += model_config._AUDIO_OFFLINE_MISS_TTL_S + 1
assert model_config.detect_audio_type_checked("org/m", local_files_only = True) == ("snac", True)
assert model_config._audio_offline_miss_cache == {}
def test_an_online_transient_failure_still_retries_immediately(monkeypatch):
"""The bound is deliberately only for probes that touched no network. A gated repo or
a 5xx must not be remembered, or fixing the token would take a minute to take."""
from utils.models import model_config
monkeypatch.setattr(model_config, "_audio_detection_cache", {})
monkeypatch.setattr(model_config, "_audio_offline_miss_cache", {})
probes = []
monkeypatch.setattr(
model_config,
"_detect_audio_from_tokenizer",
lambda name, token = None, **kw: (probes.append(name), (None, False))[1],
)
for _ in range(3):
model_config.detect_audio_type_checked("org/gated", local_files_only = False)
assert len(probes) == 3, probes
def test_every_pattern_has_a_marker_so_the_parse_can_be_skipped():
"""The marker list lets a large text tokenizer_config skip parsing; markers cannot be derived
from the lambda patterns, so a codec needs a marker or goes undetected."""
from utils.audio_tokens import AUDIO_TOKEN_MARKERS, may_hold_audio_tokens
assert set(AUDIO_TOKEN_PATTERNS) == {"csm", "whisper", "bicodec", "dac", "snac", "audio_vlm"}
samples = {
"csm": ["<|AUDIO|>", "<|audio_eos|>"],
"whisper": ["<|startoftranscript|>"],
"bicodec": ["<|bicodec_semantic_0|>"],
"dac": ["<|audio_start|>", "<|audio_end|>", "<|text_start|>", "<|text_end|>"],
"snac": [f"<custom_token_{i}>" for i in range(10001)],
"audio_vlm": ["<audio_soft_token>"],
}
for audio_type, tokens in samples.items():
assert _classify(tokens) == audio_type, audio_type
assert may_hold_audio_tokens(json.dumps(tokens)), audio_type
assert may_hold_audio_tokens(json.dumps(["<|image|>", "<|audio|>"]))
assert not may_hold_audio_tokens(
json.dumps([f"<|extra_token_{i}|>" for i in range(500)] + ["<bos>", "<eos>"])
)
assert all(marker in "".join(AUDIO_TOKEN_MARKERS) for marker in AUDIO_TOKEN_MARKERS)
def test_a_large_text_tokenizer_is_not_parsed(monkeypatch, tmp_path):
"""Ordinary tokenizer_configs are never json.loads-parsed, the bulk of a cold /loras scan."""
import json as json_module
from utils.models import model_config
config = {
"added_tokens_decoder": {
str(i): {"content": f"<|extra_token_{i}|>", "special": True} for i in range(5000)
}
}
checkpoint = tmp_path / "run"
checkpoint.mkdir()
(checkpoint / "tokenizer_config.json").write_text(json_module.dumps(config))
parsed = []
real_loads = model_config.json.loads
monkeypatch.setattr(
model_config.json,
"loads",
lambda raw, *a, **kw: (parsed.append(len(raw)), real_loads(raw, *a, **kw))[1],
)
result, definitive = model_config._detect_audio_from_tokenizer(
str(checkpoint), local_files_only = True
)
assert result is None
assert definitive is True
assert parsed == [], parsed
def test_a_half_written_tokenizer_stays_unknown(tmp_path):
"""The skip-the-parse path must not cache a part-written tokenizer as a definitive 'not audio'
for the process; it stays unknown."""
from utils.models import model_config
checkpoint = tmp_path / "mid_write"
checkpoint.mkdir()
whole = json.dumps({"added_tokens_decoder": {"0": {"content": "<|plain|>"}}})
(checkpoint / "tokenizer_config.json").write_text(whole[: len(whole) // 2])
result, definitive = model_config._detect_audio_from_tokenizer(
str(checkpoint), local_files_only = True
)
assert result is None
assert definitive is False
(checkpoint / "tokenizer_config.json").write_text(whole)
assert model_config._detect_audio_from_tokenizer(str(checkpoint), local_files_only = True) == (
None,
True,
)
def _cached_snapshot(
tmp_path,
repo_id,
files,
sha = "abc123",
):
"""A repo laid out the way the HF hub cache lays one out."""
repo_dir = tmp_path / ("models--" + repo_id.replace("/", "--"))
snapshot = repo_dir / "snapshots" / sha
snapshot.mkdir(parents = True)
for name, text in files.items():
target = snapshot / name
target.parent.mkdir(parents = True, exist_ok = True)
target.write_text(text, encoding = "utf-8")
return repo_dir, snapshot
def _detect_against_cache(
monkeypatch,
snapshot_repo_dir,
*,
sha = "abc123",
listed = ("tokenizer_config.json",),
responses = None,
**kwargs,
):
"""Returns (result, file_reads, document_reads); listed=None means the repo document is unavailable."""
import types as _types
from utils.models import model_config as mc
reads: list = []
documents: list = []
monkeypatch.setattr(mc, "_audio_detection_cache", {})
monkeypatch.setattr(mc, "_audio_offline_miss_cache", {})
monkeypatch.setattr(mc, "get_cache_path", lambda *a, **k: snapshot_repo_dir)
monkeypatch.setattr(mc, "_env_offline", lambda: False)
def _info(
model_name,
hf_token = None,
**kw,
):
documents.append(model_name)
if listed is None:
raise ConnectionError("no repo document")
return _types.SimpleNamespace(
sha = sha,
siblings = [_types.SimpleNamespace(rfilename = name) for name in listed],
)
monkeypatch.setattr(mc, "_hub_model_info", _info)
import requests
def _get(url, **kw):
reads.append(url)
return (responses or []).pop(0)
monkeypatch.setattr(requests, "get", _get)
return mc.detect_audio_type_checked("acme/tts-model", **kwargs), reads, documents
def _tokenizer(marker):
return json.dumps({"added_tokens_decoder": {"0": {"content": marker}}})
@pytest.mark.parametrize("marker, expected", [("<bos>", None), ("<|audio|>", "audio_vlm")])
def test_the_current_snapshot_answers_without_fetching_the_file(
monkeypatch, tmp_path, marker, expected
):
"""The repo document is still read; the snapshot only saves fetching the tokenizer files."""
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": _tokenizer(marker)}
)
(audio_type, definitive), reads, _documents = _detect_against_cache(monkeypatch, repo_dir)
assert audio_type == expected
assert definitive is True
assert reads == []
def test_a_tokenizer_the_repo_has_but_the_cache_lacks_still_asks_the_hub(monkeypatch, tmp_path):
"""Loadable weights do not mean every file arrived: the markers may be in the one that
did not, and a negative answer here is cached for the life of the process."""
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": _tokenizer("<bos>")}
)
(audio_type, definitive), reads, _documents = _detect_against_cache(
monkeypatch,
repo_dir,
listed = ("tokenizer_config.json", "LLM/tokenizer_config.json"),
responses = [_Resp(200, json.loads(_tokenizer("<|audio|>"))), _Resp(404)],
)
assert audio_type == "audio_vlm"
assert definitive is True
assert len(reads) == 1
def test_a_snapshot_that_is_not_the_current_commit_still_asks_the_hub(monkeypatch, tmp_path):
"""A repo re-downloaded at a new commit keeps the old snapshot beside the new one."""
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": _tokenizer("<bos>")}, sha = "old"
)
(audio_type, definitive), reads, _documents = _detect_against_cache(
monkeypatch,
repo_dir,
sha = "new",
responses = [_Resp(200, json.loads(_tokenizer("<|audio|>"))), _Resp(404)],
)
assert audio_type == "audio_vlm"
assert len(reads) == 1
def test_an_older_snapshot_beside_the_current_one_cannot_answer_negatively(monkeypatch, tmp_path):
"""Only the current commit may answer negatively, since an older snapshot predates the markers
and a negative is cached; any positive still stands."""
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": _tokenizer("<bos>")}, sha = "old"
)
current = repo_dir / "snapshots" / "new"
current.mkdir()
(current / "tokenizer_config.json").write_text(_tokenizer("<|audio|>"), encoding = "utf-8")
(audio_type, definitive), reads, _documents = _detect_against_cache(
monkeypatch, repo_dir, sha = "new"
)
assert audio_type == "audio_vlm"
assert definitive is True
assert reads == []
def test_without_a_repo_document_the_hub_still_answers(monkeypatch, tmp_path):
"""Offline, or on any failed read, there is nothing to judge the snapshot against."""
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": _tokenizer("<bos>")}
)
(audio_type, _definitive), reads, _documents = _detect_against_cache(
monkeypatch,
repo_dir,
listed = None,
responses = [_Resp(200, json.loads(_tokenizer("<|audio|>"))), _Resp(404)],
)
assert audio_type == "audio_vlm"
assert len(reads) == 1
def test_local_files_only_reads_no_repo_document(monkeypatch, tmp_path):
"""The /loras filesystem scan asks for no network, and resolving the current commit
would be one. The local read still answers it, exactly as it did before."""
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": _tokenizer("<bos>")}
)
(audio_type, definitive), reads, documents = _detect_against_cache(
monkeypatch, repo_dir, local_files_only = True
)
assert (audio_type, definitive) == (None, True)
assert reads == []
assert documents == []
def test_a_half_written_tokenizer_ending_in_a_brace_still_asks_the_hub(monkeypatch, tmp_path):
"""The trailing brace tells a whole file from a half-written one only by luck. Standing
in for the Hub copy needs more than luck, since the markers may be in the missing tail."""
whole = json.dumps(
{"added_tokens_decoder": {"0": {"content": "<bos>"}, "1": {"content": "<|audio|>"}}}
)
truncated = whole[: whole.index('"1"')] + "}"
assert truncated.rstrip().endswith("}") and "<|audio|>" not in truncated
repo_dir, _ = _cached_snapshot(tmp_path, "acme/tts-model", {"tokenizer_config.json": truncated})
(audio_type, definitive), reads, _documents = _detect_against_cache(
monkeypatch, repo_dir, responses = [_Resp(200, json.loads(whole)), _Resp(404)]
)
assert audio_type == "audio_vlm"
assert definitive is True
assert len(reads) == 1
def test_a_tokenizer_that_is_unreadable_still_asks_the_hub(monkeypatch, tmp_path):
"""Nothing was read, so having the file says nothing: a truncated file is not a negative."""
whole = _tokenizer("<|audio|>")
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": whole[: len(whole) // 2]}
)
(audio_type, _definitive), reads, _documents = _detect_against_cache(
monkeypatch, repo_dir, responses = [_Resp(200, json.loads(whole)), _Resp(404)]
)
assert audio_type == "audio_vlm"
assert len(reads) == 1
def test_a_document_that_lists_no_files_cannot_answer_negatively(monkeypatch, tmp_path):
"""A repo document listing no files must not answer negatively; the result is cached for the process."""
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": _tokenizer("<bos>")}
)
(audio_type, definitive), reads, _documents = _detect_against_cache(
monkeypatch,
repo_dir,
listed = (),
responses = [_Resp(200, json.loads(_tokenizer("<|audio|>"))), _Resp(404)],
)
assert audio_type == "audio_vlm"
assert definitive is True
assert len(reads) == 1
def test_a_document_without_siblings_at_all_cannot_answer_negatively(monkeypatch, tmp_path):
"""A missing siblings key is a valid hub response shape, distinct from an empty list."""
import types as _types
from utils.models import model_config as mc
repo_dir, _ = _cached_snapshot(
tmp_path, "acme/tts-model", {"tokenizer_config.json": _tokenizer("<bos>")}
)
monkeypatch.setattr(mc, "get_cache_path", lambda *a, **k: repo_dir)
monkeypatch.setattr(
mc,
"_hub_model_info",
lambda *a, **k: _types.SimpleNamespace(sha = "abc123", siblings = None),
)
assert mc._current_cached_snapshot("acme/tts-model") is None
def test_a_marker_written_as_an_escape_is_still_found(monkeypatch, tmp_path):
"""Markers written as JSON unicode escapes must still be found, or a miss gets cached as negative."""
from utils.models.model_config import _may_hold_audio_tokens
escaped = (
_tokenizer("<|audio|>").replace("<", chr(92) + "u003c").replace(">", chr(92) + "u003e")
)
assert "<|audio|>" not in escaped and json.loads(escaped)
assert not _may_hold_audio_tokens(escaped), "the raw scan is what misses it"
repo_dir, _ = _cached_snapshot(tmp_path, "acme/tts-model", {"tokenizer_config.json": escaped})
(audio_type, definitive), reads, _documents = _detect_against_cache(monkeypatch, repo_dir)
assert (audio_type, definitive) == ("audio_vlm", True)
assert reads == [], "and the snapshot still answers it without a fetch"
_LFM3_OMNI_TOKENIZER_CONFIG = {
"backend": "tokenizers",
"image_token": "<|image|>",
"processor_class": "Lfm3OmniProcessor",
"tokenizer_class": "TokenizersBackend",
"video_token": "<|video|>",
}
_LFM3_OMNI_PROCESSOR_CONFIG = {
"audio_token": "<|audio|>",
"audio_out_token": "<|audio_out|>",
"image_processor": {"image_processor_type": "Lfm3OmniImageProcessor"},
"image_token": "<|image|>",
"processor_class": "Lfm3OmniProcessor",
}
_LFM3_AUDIO_PROCESSOR_CONFIG = {
"audio_out_token": "<|audio_out|>",
"audio_token": "<|audio|>",
"feature_extractor": {"feature_extractor_type": "Lfm3AudioFeatureExtractor"},
"processor_class": "Lfm3AudioProcessor",
}
def test_a_processor_audio_token_is_audio_vlm():
from utils.audio_tokens import classify_audio_tokens
assert classify_audio_tokens(_LFM3_OMNI_TOKENIZER_CONFIG) is None
assert classify_audio_tokens(_LFM3_OMNI_PROCESSOR_CONFIG) == "audio_vlm"
# Only the markers the tokenizer patterns already accept: Qwen2-Audio's <|AUDIO|> stays unrouted.
assert (
classify_audio_tokens(
{
"audio_token": "<|AUDIO|>",
"image_processor": {"image_processor_type": "Lfm3ImageProcessor"},
}
)
is None
)
assert classify_audio_tokens({"audio_token": None}) is None
def test_a_local_lfm3_omni_checkpoint_takes_audio_input(tmp_path):
(tmp_path / "config.json").write_text(json.dumps({"model_type": "lfm3_omni"}))
(tmp_path / "tokenizer_config.json").write_text(json.dumps(_LFM3_OMNI_TOKENIZER_CONFIG))
(tmp_path / "processor_config.json").write_text(json.dumps(_LFM3_OMNI_PROCESSOR_CONFIG))
audio_type, definitive = detect_audio_type_checked(str(tmp_path))
assert (audio_type, definitive) == ("audio_vlm", True)
assert is_audio_input_type(audio_type)
def test_lfm3_omni_on_the_hub_is_audio_vlm(monkeypatch):
audio_type, definitive = _detect_checked(
monkeypatch,
[
_Resp(200, _LFM3_OMNI_TOKENIZER_CONFIG),
_Resp(404),
_Resp(200, _LFM3_OMNI_PROCESSOR_CONFIG),
],
model = "LiquidAI/LFM3-Omni-2B",
)
assert (audio_type, definitive) == ("audio_vlm", True)
def test_an_audio_feature_extractor_processor_is_audio_vlm():
from utils.audio_tokens import classify_audio_tokens
assert classify_audio_tokens(_LFM3_AUDIO_PROCESSOR_CONFIG) == "audio_vlm"
def test_an_audio_token_with_no_input_processor_is_not_audio_vlm():
from utils.audio_tokens import classify_audio_tokens
bare = {k: v for k, v in _LFM3_AUDIO_PROCESSOR_CONFIG.items() if k != "feature_extractor"}
assert classify_audio_tokens(bare) is None
def test_lfm3_audio_on_the_hub_is_audio_vlm(monkeypatch):
audio_type, definitive = _detect_checked(
monkeypatch,
[
_Resp(200, _LFM3_OMNI_TOKENIZER_CONFIG),
_Resp(404),
_Resp(200, _LFM3_AUDIO_PROCESSOR_CONFIG),
],
model = "LiquidAI/LFM3-Audio-2B",
)
assert (audio_type, definitive) == ("audio_vlm", True)
# Gemma 4's transformers 5 tokenizer_config: no added_tokens_decoder, a named audio_token. Every size
# carries it; only E2B/E4B/12B have an audio encoder.
_V5_AUDIO_TOKENIZER = {"audio_token": "<|audio|>", "image_token": "<|image|>"}
_WITH_ENCODER = {"model_type": "gemma4", "audio_config": {"model_type": "gemma4_audio"}}
_WITHOUT_ENCODER = {"model_type": "gemma4", "audio_config": None}
def _escaped(tok_config):
return json.dumps(tok_config).replace("<", chr(92) + "u003c").replace(">", chr(92) + "u003e")
@pytest.mark.parametrize(
"tok_config, model_config, expected",
[
(_V5_AUDIO_TOKENIZER, _WITH_ENCODER, ("audio_vlm", True)),
(_V5_AUDIO_TOKENIZER, _WITHOUT_ENCODER, (None, True)),
(_V5_AUDIO_TOKENIZER, None, (None, False)),
({"image_token": "<|image|>"}, _WITH_ENCODER, (None, True)),
(
{**_V5_AUDIO_TOKENIZER, "added_tokens_decoder": {"0": {"content": "<|bicodec_x|>"}}},
_WITH_ENCODER,
("bicodec", True),
),
],
)
def test_a_declared_audio_token_needs_an_audio_encoder(tok_config, model_config, expected):
from utils.models.model_config import _classify_audio_tokenizer
assert _classify_audio_tokenizer(tok_config, lambda: model_config) == expected
@pytest.mark.parametrize(
"tokenizer_text", [json.dumps(_V5_AUDIO_TOKENIZER), _escaped(_V5_AUDIO_TOKENIZER)]
)
@pytest.mark.parametrize(
"model_config, expected", [(_WITH_ENCODER, "audio_vlm"), (_WITHOUT_ENCODER, None)]
)
def test_a_cached_gemma4_snapshot_is_typed_by_its_audio_encoder(
monkeypatch, tmp_path, tokenizer_text, model_config, expected
):
repo_dir, _ = _cached_snapshot(
tmp_path,
"acme/tts-model",
{"tokenizer_config.json": tokenizer_text, "config.json": json.dumps(model_config)},
)
(audio_type, definitive), reads, _documents = _detect_against_cache(monkeypatch, repo_dir)
assert (audio_type, definitive, reads) == (expected, True, [])
@pytest.mark.parametrize(
"tokenizer_text", [json.dumps(_V5_AUDIO_TOKENIZER), _escaped(_V5_AUDIO_TOKENIZER)]
)
@pytest.mark.parametrize("config_text", ["{half", None])
def test_a_cached_config_it_cannot_read_leaves_the_hub_to_answer(
monkeypatch, tmp_path, tokenizer_text, config_text
):
# None: a tokenizer-only download, whose snapshot never fetched config.json.
files = {"tokenizer_config.json": tokenizer_text}
if config_text is not None:
files["config.json"] = config_text
repo_dir, _ = _cached_snapshot(tmp_path, "acme/tts-model", files)
(audio_type, _definitive), reads, _documents = _detect_against_cache(
monkeypatch,
repo_dir,
listed = ("tokenizer_config.json", "config.json"),
responses = [_Resp(200, _V5_AUDIO_TOKENIZER), _Resp(200, _WITH_ENCODER)],
)
assert audio_type == "audio_vlm"
assert reads[-1].endswith("/config.json")
def test_a_cached_adapter_repo_answers_without_the_hub(monkeypatch, tmp_path):
# An adapter repo never ships config.json, so its absence is not a partial download.
repo_dir, _ = _cached_snapshot(
tmp_path,
"acme/tts-model",
{"tokenizer_config.json": json.dumps(_V5_AUDIO_TOKENIZER), "adapter_config.json": "{}"},
)
(audio_type, definitive), reads, _documents = _detect_against_cache(
monkeypatch, repo_dir, listed = ("tokenizer_config.json", "adapter_config.json")
)
assert (audio_type, definitive, reads) == (None, True, [])
def test_a_local_folder_without_a_config_is_definitively_not_audio(tmp_path):
from utils.models import model_config
(tmp_path / "tokenizer_config.json").write_text(json.dumps(_V5_AUDIO_TOKENIZER))
assert model_config._detect_audio_from_tokenizer(str(tmp_path), local_files_only = True) == (
None,
True,
)
@pytest.mark.parametrize(
"config_response, expected",
[
(_Resp(200, _WITH_ENCODER), ("audio_vlm", True)),
(_Resp(200, _WITHOUT_ENCODER), (None, True)),
(_Resp(404), (None, True)),
(_Resp(503), (None, False)),
],
)
def test_the_hub_config_decides_a_declared_audio_token(monkeypatch, config_response, expected):
responses = [_Resp(200, _V5_AUDIO_TOKENIZER), config_response, _Resp(404)]
assert _detect_checked(monkeypatch, responses) == expected
def test_an_unread_encoder_config_stays_unknown_past_a_readable_processor(monkeypatch):
# Gemma 4's processor_config carries no top-level audio_token, so it must not settle the answer.
responses = [
_Resp(200, _V5_AUDIO_TOKENIZER),
_Resp(503),
_Resp(404),
_Resp(200, {"feature_extractor": {}}),
]
assert _detect_checked(monkeypatch, responses) == (None, False)
def test_a_local_folder_with_an_escaped_audio_token_reads_its_encoder(tmp_path):
from utils.models import model_config
(tmp_path / "tokenizer_config.json").write_text(_escaped(_V5_AUDIO_TOKENIZER))
(tmp_path / "config.json").write_text(json.dumps(_WITH_ENCODER))
assert model_config._detect_audio_from_tokenizer(str(tmp_path), local_files_only = True) == (
"audio_vlm",
True,
)