Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#265 — live-codec TTS audio encoding (Orpheus/SNAC + Llasa/XCodec2).
Lifts the v0.71.20 ``data.format='audio'`` hardware gate into a real
encode-at-train-time path. Rows shaped ``{"audio": <path>, "messages": [...]}``
(the v0.17.0 audio format — paths are resolved + containment-checked by the
data loader) are encoded into discrete codec-token strings that become the
assistant turn, after which training proceeds through the validated
pre-encoded SFT cross-entropy path.
The generic pipeline (``encode_tts_dataset`` → ``encode_tts_row`` →
per-family encoder) is family-agnostic. It now has codec-string encoders for:
* **Orpheus / SNAC** (``pip install snac``) — 24 kHz SNAC produces 3
codebooks at a 1/2/4 frame ratio; Orpheus interleaves them 7 tokens per
frame with per-slot codebook offsets and renders each code as
``<custom_token_N>`` where ``N = code + 10 + slot*4096`` (the official
Orpheus id layout: ``id = 128256 + N = 128266 + code + slot*4096`` on the
Orpheus tokenizer, whose ``<custom_token_i>`` maps to ``128256 + i``).
* **Llasa / XCodec2** — 16 kHz XCodec2 ids are rendered as Llasa speech
tokens between its native speech-generation boundary tokens.
Spark and Oute raw-audio live encoding is deliberately refused on this path
because their current upstream stacks are incompatible with Soup. Sesame CSM
is refused on this codec-string path: its native trainer uses 32 Mimi codebooks plus text as
parallel multimodal frames, which cannot be represented faithfully by replacing
a text assistant turn. Raw-audio transcript pairs use the dedicated native CSM
trainer instead.
Security / robustness:
- Heavy imports (numpy / soundfile / torch / snac / transformers) are lazy — module import
stays light for the CLI hot path.
- Audio paths: null-byte rejection, symlink rejection (``os.lstat``,
defence-in-depth — the loader already containment-checks under
``data.audio_dir``), duration cap (``_MAX_AUDIO_SECONDS``).
- Pure interleave kernel validates the 1/2/4 codebook ratio, per-code bounds
``[0, 4096)``, and rejects bool codes (project bool-as-int policy).
- Rows are deep-copied — the caller's dataset is never mutated (v0.33.0 #47
immutability policy).
"""
from __future__ import annotations
import os
import stat
from typing import Any, Callable, Optional
from soup_cli.utils.paths import is_device_namespace_path
from soup_cli.utils.tts import validate_tts_family
# SNAC checkpoint for the Orpheus family (24 kHz, 3 codebooks).
SNAC_MODEL_ID = "hubertsiuzdak/snac_24khz"
SNAC_SAMPLE_RATE = 24_000
# Orpheus token layout: <custom_token_N> with N = code + OFFSET + slot*4096.
# Slots 0..9 are Orpheus control tokens; audio codes start at 10.
ORPHEUS_CODE_OFFSET = 10
ORPHEUS_CODEBOOK_SIZE = 4096
_ORPHEUS_FRAME_SLOTS = 7
# Families whose live-codec encoder is implemented AND validated.
LIVE_CODEC_FAMILIES: frozenset[str] = frozenset({"orpheus", "llasa"})
_MAX_AUDIO_SECONDS = 600.0 # 10 min cap — defends against runaway encodes
# Byte cap (~600 s of 24 kHz stereo float32 + headroom) — rejected before any
# read so a runaway file never materialises into RAM.
_MAX_AUDIO_BYTES = 512 * 1024 * 1024
# Process-level SNAC model cache (one load per device). NOTE: single-threaded
# assumption — the live-codec trainer encode path runs on one thread, so this
# unbounded, unguarded cache is intentional (no eviction / no lock needed).
_SNAC_CACHE: dict[str, Any] = {}
_XCODEC2_CACHE: dict[str, Any] = {}
XCODEC2_MODEL_ID = "HKUSTAudio/xcodec2-hf"
XCODEC2_SAMPLE_RATE = 16_000
XCODEC2_CODEBOOK_SIZE = 65_536
# ---------------------------------------------------------------------------
# Pure interleave kernel (no torch — unit-testable on plain lists)
# ---------------------------------------------------------------------------
def _check_codes(codes, name: str, expected_len: int) -> list[int]:
out = []
for code in codes:
if isinstance(code, bool) or not isinstance(code, int):
raise TypeError(f"{name} codes must be non-bool ints")
if not (0 <= code < ORPHEUS_CODEBOOK_SIZE):
raise ValueError(
f"{name} code {code} out of range [0, {ORPHEUS_CODEBOOK_SIZE})"
)
out.append(code)
if len(out) != expected_len:
raise ValueError(
f"{name} codebook length {len(out)} != expected {expected_len} "
"(SNAC codebooks must have the 1/2/4 frame ratio)"
)
return out
def interleave_orpheus_codes(coarse, medium, fine) -> list[int]:
"""Interleave the 3 SNAC codebooks into Orpheus token indices.
``coarse``/``medium``/``fine`` are the per-frame code sequences at the
SNAC 1/2/4 ratio (lengths T / 2T / 4T). Returns 7 token indices per
frame in the official Orpheus slot order, each offset by
``ORPHEUS_CODE_OFFSET + slot * ORPHEUS_CODEBOOK_SIZE`` so the rendered
``<custom_token_N>`` strings map onto the Orpheus tokenizer's audio ids.
"""
coarse = list(coarse)
if not coarse:
raise ValueError("coarse codebook must be non-empty")
frames = len(coarse)
coarse = _check_codes(coarse, "coarse", frames)
medium = _check_codes(list(medium), "medium", 2 * frames)
fine = _check_codes(list(fine), "fine", 4 * frames)
size = ORPHEUS_CODEBOOK_SIZE
base = ORPHEUS_CODE_OFFSET
out: list[int] = []
for i in range(frames):
out.append(coarse[i] + base)
out.append(medium[2 * i] + base + size)
out.append(fine[4 * i] + base + 2 * size)
out.append(fine[4 * i + 1] + base + 3 * size)
out.append(medium[2 * i + 1] + base + 4 * size)
out.append(fine[4 * i + 2] + base + 5 * size)
out.append(fine[4 * i + 3] + base + 6 * size)
return out
def orpheus_tokens_to_string(indices) -> str:
"""Render Orpheus token indices as a ``<custom_token_N>`` string.
Indices must be non-bool ints (mirrors :func:`_check_codes` — project
bool-as-int policy; no float/bool coercion).
"""
indices = list(indices)
if not indices:
raise ValueError("token index list must be non-empty")
for n in indices:
if isinstance(n, bool) or not isinstance(n, int):
raise TypeError("token indices must be non-bool ints")
return "".join(f"<custom_token_{n}>" for n in indices)
# ---------------------------------------------------------------------------
# Audio loading (lazy soundfile / numpy)
# ---------------------------------------------------------------------------
def load_audio_mono(path: str, *, target_sr: int = SNAC_SAMPLE_RATE):
"""Load an audio file as mono float32 at ``target_sr``.
Lazy-imports soundfile (friendly ImportError naming the install).
Off-rate audio is resampled with basic linear interpolation — adequate
for codec encoding of speech; operators wanting studio-grade resampling
should resample offline. Returns a 1-D ``np.float32`` array.
Only a regular file is read: a directory, a FIFO or a device raises
``ValueError`` before ``soundfile`` sees it. The check is made before
``soundfile`` opens the path, so a path replaced in between is not caught.
"""
if not isinstance(path, str) or not path:
raise ValueError("audio path must be a non-empty string")
if "\x00" in path:
raise ValueError("audio path must not contain null bytes")
if is_device_namespace_path(path):
raise ValueError("audio path must not be a device path")
if not os.path.exists(path):
raise FileNotFoundError(
f"audio file not found: {os.path.basename(path)!r}"
)
# Symlink rejection — defence-in-depth; the loader already containment-
# checks rows under data.audio_dir (v0.17.0 policy).
mode = os.lstat(path).st_mode
if stat.S_ISLNK(mode):
raise ValueError("audio path must not be a symlink")
# A FIFO would block soundfile, and a device would never end.
if not stat.S_ISREG(mode):
raise ValueError("audio path must be a regular file")
# Byte-size cap before any read — defends against a runaway file.
if os.path.getsize(path) > _MAX_AUDIO_BYTES:
raise ValueError(
f"audio file exceeds the {_MAX_AUDIO_BYTES} byte live-codec cap; "
"split the clip or pre-encode offline"
)
try:
import soundfile
except ImportError as exc: # pragma: no cover - dep present in CI
raise ImportError(
"Live-codec TTS needs the 'soundfile' package to read audio. "
"Install it with `pip install soundfile` (or `pip install "
"'soup-cli[audio]'`)."
) from exc
import numpy as np
# Probe metadata FIRST — reject an over-long clip before materialising the
# whole file into RAM (the post-read check below stays as defence-in-depth).
info = soundfile.info(path)
if info.samplerate and info.frames / float(info.samplerate) > _MAX_AUDIO_SECONDS:
raise ValueError(
f"audio is {info.frames / float(info.samplerate):.1f} seconds — "
f"exceeds the {_MAX_AUDIO_SECONDS:.0f} seconds live-codec cap; "
"split the clip or pre-encode offline"
)
# Open via O_NOFOLLOW and hand the fd to soundfile — closes the TOCTOU
# window between the lstat symlink check above and the read (project
# v0.65/v0.67 O_NOFOLLOW reader policy). soundfile.read accepts an open
# file object.
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
with os.fdopen(fd, "rb") as handle:
data, sr = soundfile.read(handle, dtype="float32", always_2d=True)
except OSError as exc:
# O_NOFOLLOW raises ELOOP if the path became a symlink after the lstat.
raise ValueError(f"audio path could not be read: {exc}") from exc
duration = data.shape[0] / float(sr)
if duration > _MAX_AUDIO_SECONDS:
raise ValueError(
f"audio is {duration:.1f} seconds — exceeds the "
f"{_MAX_AUDIO_SECONDS:.0f} seconds live-codec cap; split the "
"clip or pre-encode offline"
)
mono = data.mean(axis=1)
if int(sr) != int(target_sr):
n_out = max(1, int(round(len(mono) * target_sr / float(sr))))
x_old = np.linspace(0.0, 1.0, num=len(mono), endpoint=False)
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
mono = np.interp(x_new, x_old, mono)
return mono.astype(np.float32)
# ---------------------------------------------------------------------------
# Orpheus / SNAC encoder
# ---------------------------------------------------------------------------
def _get_snac_model(device: Optional[str] = None):
"""Load (once per device) the SNAC codec model. Lazy snac import."""
dev = device or "cpu"
cached = _SNAC_CACHE.get(dev)
if cached is not None:
return cached
try:
from snac import SNAC
except ImportError as exc:
raise ImportError(
"TTS family 'orpheus' live-codec training requires the 'snac' "
"package (audio codec). Install it with `pip install snac`, or "
"pre-encode your audio to codec tokens offline and train with "
"data.format=chatml."
) from exc
model = SNAC.from_pretrained(SNAC_MODEL_ID).eval().to(dev)
_SNAC_CACHE[dev] = model
return model
def encode_audio_orpheus(
path: str,
*,
snac_model: Optional[Any] = None,
device: Optional[str] = None,
) -> str:
"""Encode one audio file into the Orpheus ``<custom_token_N>`` string.
``snac_model`` is injectable (tests / pre-loaded models); when ``None``
the 24 kHz SNAC checkpoint is loaded once per process per device.
"""
audio = load_audio_mono(path, target_sr=SNAC_SAMPLE_RATE)
import torch
model = snac_model if snac_model is not None else _get_snac_model(device)
wav = torch.from_numpy(audio).reshape(1, 1, -1)
if device:
wav = wav.to(device)
with torch.no_grad():
codes = model.encode(wav)
coarse = [int(c) for c in codes[0][0].tolist()]
medium = [int(c) for c in codes[1][0].tolist()]
fine = [int(c) for c in codes[2][0].tolist()]
return orpheus_tokens_to_string(
interleave_orpheus_codes(coarse, medium, fine)
)
def llasa_codes_to_string(codes) -> str:
"""Render XCodec2 ids in Llasa's native speech-token envelope."""
values = list(codes)
if not values:
raise ValueError("Llasa XCodec2 code list must be non-empty")
rendered = []
for code in values:
if isinstance(code, bool) or not isinstance(code, int):
raise TypeError("Llasa XCodec2 codes must be non-bool ints")
if not (0 <= code < XCODEC2_CODEBOOK_SIZE):
raise ValueError(
f"Llasa XCodec2 code {code} out of range "
f"[0, {XCODEC2_CODEBOOK_SIZE})"
)
rendered.append(f"<|s_{code}|>")
return (
"<|SPEECH_GENERATION_START|>"
+ "".join(rendered)
+ "<|SPEECH_GENERATION_END|>"
)
def _get_xcodec2_components(device: Optional[str] = None):
"""Load Transformers-native XCodec2 lazily at Soup's dependency floor."""
dev = device or "cpu"
cached = _XCODEC2_CACHE.get(dev)
if cached is not None:
return cached
try:
from transformers import AutoFeatureExtractor, Xcodec2Model
except ImportError as exc:
raise RuntimeError(
"Llasa live-codec requires Transformers with native XCodec2 support "
"(Soup requires transformers>=5.16.1). Upgrade the train extra rather "
"than installing the legacy xcodec2 package, which pins an old Torch."
) from exc
extractor = AutoFeatureExtractor.from_pretrained(XCODEC2_MODEL_ID)
model = Xcodec2Model.from_pretrained(XCODEC2_MODEL_ID).eval().to(dev)
_XCODEC2_CACHE[dev] = (model, extractor)
return model, extractor
def clear_xcodec2_cache(device: Optional[str] = None) -> None:
"""Release the cached XCodec2 model after dataset encoding (#1112 review)."""
dev = device or "cpu"
_XCODEC2_CACHE.pop(dev, None)
# The live-codec pass runs before the base model/trainer is materialised.
# Do not keep a ~2.5 GB fp32 XCodec2 checkpoint resident for the entire run.
import gc
gc.collect()
if str(dev).startswith("cuda"):
try:
import torch
except ImportError: # pragma: no cover - train extra always has torch
return
if torch.cuda.is_available():
torch.cuda.empty_cache()
def encode_audio_llasa(
path: str,
*,
xcodec2_model: Optional[Any] = None,
feature_extractor: Optional[Any] = None,
device: Optional[str] = None,
) -> str:
"""Encode 16 kHz speech with Transformers-native XCodec2."""
audio = load_audio_mono(path, target_sr=XCODEC2_SAMPLE_RATE)
import torch
if (xcodec2_model is None) != (feature_extractor is None):
raise ValueError("xcodec2_model and feature_extractor must be provided together")
if xcodec2_model is None:
model, extractor = _get_xcodec2_components(device)
else:
model, extractor = xcodec2_model, feature_extractor
inputs = extractor(
audio=audio,
sampling_rate=XCODEC2_SAMPLE_RATE,
return_tensors="pt",
)
if device:
inputs = {
key: value.to(device) if hasattr(value, "to") else value
for key, value in inputs.items()
}
with torch.inference_mode():
encoded = model.encode(**inputs)
codes = encoded.audio_codes
values = [int(v) for v in codes.reshape(-1).detach().cpu().tolist()]
return llasa_codes_to_string(values)
def csm_live_codec_error() -> RuntimeError:
return RuntimeError(
"TTS family 'sesame_csm' cannot use Soup's codec-string SFT "
"live-codec path: CSM trains 32 Mimi codebooks plus text as parallel "
"multimodal frames, not audio ids embedded in a text assistant turn. "
"Use Soup's dedicated CSM trainer with training.tts_family=sesame_csm, "
"data.format=audio and training.quantization=none. Pre-encoded "
"data.format=chatml is not a native CSM training example."
)
def incompatible_live_codec_error(family: str) -> RuntimeError:
"""Explain upstream packaging/objective blockers for #265 families."""
canonical = validate_tts_family(family)
if canonical == "spark":
return RuntimeError(
"TTS family 'spark' has no installable 'sparktts' PyPI package. "
"The official Spark-TTS source pins torch==2.5.1, torchaudio==2.5.1 "
"and transformers==4.46.2, which conflicts with Soup's supported "
"training stack (#265). Refusing raw-audio live encoding instead of "
"suggesting a nonexistent install or downgrading the environment. "
"Pre-encode with the official Spark-TTS environment and train with "
"data.format=chatml until a compatible native codec integration lands."
)
if canonical == "oute":
return RuntimeError(
"TTS family 'oute' live-codec preparation cannot safely depend on "
"the current outetts package: its published metadata pins "
"transformers==4.52.3 while Soup requires transformers>=5.16.1. "
"Oute v0.3 training also needs transcript/word alignment plus audio "
"codes, not a bare WAV-to-token substitution (#265). Refusing raw-audio "
"live encoding rather than silently downgrading Transformers or "
"training the wrong target. Use an upstream Oute environment to "
"pre-encode training examples and train with data.format=chatml "
"until a compatible native path lands."
)
return RuntimeError(
f"TTS family {canonical!r} has no incompatible-live-codec contract"
)
def tts_encoder_for_family(
family: str, *, device: Optional[str] = None
) -> Callable[[str], str]:
"""Return the live audio→codec-string encoder for ``family``.
Only the families in :data:`LIVE_CODEC_FAMILIES` have a codec-string
encoder. Orpheus uses SNAC and Llasa uses XCodec2. Spark/Oute raw-audio
encoding is refused for upstream-stack incompatibility; Sesame CSM is
refused because its parallel Mimi-codebook objective needs a dedicated
multimodal trainer rather than this text-SFT adapter.
"""
canonical = validate_tts_family(family)
if canonical == "orpheus":
def _encode(path: str) -> str:
return encode_audio_orpheus(path, device=device)
return _encode
if canonical == "llasa":
def _encode(path: str) -> str:
return encode_audio_llasa(path, device=device)
return _encode
if canonical == "sesame_csm":
raise csm_live_codec_error()
if canonical in {"spark", "oute"}:
raise incompatible_live_codec_error(canonical)
raise AssertionError(f"unhandled TTS family after validation: {canonical!r}")
# ---------------------------------------------------------------------------
# Dataset mapping
# ---------------------------------------------------------------------------
def encode_tts_row(row: dict, encoder: Callable[[str], str]) -> dict:
"""Encode one ``{"audio", "messages"}`` row into a chat row.
The codec-token string becomes the FINAL assistant turn: an existing
trailing assistant message has its content replaced (the audio is the
training target), otherwise an assistant turn is appended. Returns a NEW
deep-copied row without the ``audio`` key — the caller's row is never
mutated.
"""
import copy
if not isinstance(row, dict):
raise TypeError(f"row must be a dict, got {type(row).__name__}")
audio = row.get("audio")
if not isinstance(audio, str) or not audio:
raise ValueError("live-codec TTS row must have a non-empty 'audio' path")
messages = row.get("messages")
if not isinstance(messages, (list, tuple)):
raise ValueError("live-codec TTS row must have a 'messages' list")
codec_string = encoder(audio)
new_messages = [copy.deepcopy(m) for m in messages]
if new_messages and isinstance(new_messages[-1], dict) and (
new_messages[-1].get("role") == "assistant"
):
new_messages[-1]["content"] = codec_string
else:
new_messages.append({"role": "assistant", "content": codec_string})
out = {
key: copy.deepcopy(value)
for key, value in row.items()
if key not in ("audio", "messages")
}
out["messages"] = new_messages
return out
def encode_tts_dataset(
dataset: dict,
family: str,
*,
encoder: Optional[Callable[[str], str]] = None,
device: Optional[str] = None,
console: Optional[Any] = None,
) -> dict:
"""Encode every ``train`` / ``val`` row's audio into codec-token chat rows.
Returns a NEW dataset dict (input never mutated). ``encoder`` is
injectable for tests; by default it is resolved per-family via
:func:`tts_encoder_for_family` (which raises for not-yet-implemented
families — #265).
"""
if not isinstance(dataset, dict):
raise TypeError(
f"dataset must be a dict, got {type(dataset).__name__}"
)
canonical = validate_tts_family(family)
encode = encoder if encoder is not None else tts_encoder_for_family(
canonical, device=device
)
new_dataset = dict(dataset)
total = 0
for split in ("train", "val"):
rows = new_dataset.get(split)
if not isinstance(rows, (list, tuple)):
continue
new_dataset[split] = [encode_tts_row(row, encode) for row in rows]
total += len(rows)
if console is not None:
console.print(
f"[green]TTS live-codec:[/] encoded {total} row(s) "
f"(family={canonical})"
)
return new_dataset