Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

21.5 KB
Raw Blame
"""#265 — live-codec TTS audio encoding (Orpheus/SNAC + Llasa/XCodec2).

Lifts the v0.71.20 ``data.format='audio'`` hardware gate into a real
encode-at-train-time path. Rows shaped ``{"audio": <path>, "messages": [...]}``
(the v0.17.0 audio format — paths are resolved + containment-checked by the
data loader) are encoded into discrete codec-token strings that become the
assistant turn, after which training proceeds through the validated
pre-encoded SFT cross-entropy path.

The generic pipeline (``encode_tts_dataset`` → ``encode_tts_row`` →
per-family encoder) is family-agnostic. It now has codec-string encoders for:

* **Orpheus / SNAC** (``pip install snac``) — 24 kHz SNAC produces 3
  codebooks at a 1/2/4 frame ratio; Orpheus interleaves them 7 tokens per
  frame with per-slot codebook offsets and renders each code as
  ``<custom_token_N>`` where ``N = code + 10 + slot*4096`` (the official
  Orpheus id layout: ``id = 128256 + N = 128266 + code + slot*4096`` on the
  Orpheus tokenizer, whose ``<custom_token_i>`` maps to ``128256 + i``).

* **Llasa / XCodec2** — 16 kHz XCodec2 ids are rendered as Llasa speech
  tokens between its native speech-generation boundary tokens.

Spark and Oute raw-audio live encoding is deliberately refused on this path
because their current upstream stacks are incompatible with Soup. Sesame CSM
is refused on this codec-string path: its native trainer uses 32 Mimi codebooks plus text as
parallel multimodal frames, which cannot be represented faithfully by replacing
a text assistant turn. Raw-audio transcript pairs use the dedicated native CSM
trainer instead.

Security / robustness:
- Heavy imports (numpy / soundfile / torch / snac / transformers) are lazy — module import
  stays light for the CLI hot path.
- Audio paths: null-byte rejection, symlink rejection (``os.lstat``,
  defence-in-depth — the loader already containment-checks under
  ``data.audio_dir``), duration cap (``_MAX_AUDIO_SECONDS``).
- Pure interleave kernel validates the 1/2/4 codebook ratio, per-code bounds
  ``[0, 4096)``, and rejects bool codes (project bool-as-int policy).
- Rows are deep-copied — the caller's dataset is never mutated (v0.33.0 #47
  immutability policy).
"""

from __future__ import annotations

import os
import stat
from typing import Any, Callable, Optional

from soup_cli.utils.paths import is_device_namespace_path
from soup_cli.utils.tts import validate_tts_family

# SNAC checkpoint for the Orpheus family (24 kHz, 3 codebooks).
SNAC_MODEL_ID = "hubertsiuzdak/snac_24khz"
SNAC_SAMPLE_RATE = 24_000

# Orpheus token layout: <custom_token_N> with N = code + OFFSET + slot*4096.
# Slots 0..9 are Orpheus control tokens; audio codes start at 10.
ORPHEUS_CODE_OFFSET = 10
ORPHEUS_CODEBOOK_SIZE = 4096
_ORPHEUS_FRAME_SLOTS = 7

# Families whose live-codec encoder is implemented AND validated.
LIVE_CODEC_FAMILIES: frozenset[str] = frozenset({"orpheus", "llasa"})

_MAX_AUDIO_SECONDS = 600.0  # 10 min cap — defends against runaway encodes
# Byte cap (~600 s of 24 kHz stereo float32 + headroom) — rejected before any
# read so a runaway file never materialises into RAM.
_MAX_AUDIO_BYTES = 512 * 1024 * 1024

# Process-level SNAC model cache (one load per device). NOTE: single-threaded
# assumption — the live-codec trainer encode path runs on one thread, so this
# unbounded, unguarded cache is intentional (no eviction / no lock needed).
_SNAC_CACHE: dict[str, Any] = {}
_XCODEC2_CACHE: dict[str, Any] = {}

XCODEC2_MODEL_ID = "HKUSTAudio/xcodec2-hf"
XCODEC2_SAMPLE_RATE = 16_000
XCODEC2_CODEBOOK_SIZE = 65_536


# ---------------------------------------------------------------------------
# Pure interleave kernel (no torch — unit-testable on plain lists)
# ---------------------------------------------------------------------------


def _check_codes(codes, name: str, expected_len: int) -> list[int]:
    out = []
    for code in codes:
        if isinstance(code, bool) or not isinstance(code, int):
            raise TypeError(f"{name} codes must be non-bool ints")
        if not (0 <= code < ORPHEUS_CODEBOOK_SIZE):
            raise ValueError(
                f"{name} code {code} out of range [0, {ORPHEUS_CODEBOOK_SIZE})"
            )
        out.append(code)
    if len(out) != expected_len:
        raise ValueError(
            f"{name} codebook length {len(out)} != expected {expected_len} "
            "(SNAC codebooks must have the 1/2/4 frame ratio)"
        )
    return out


def interleave_orpheus_codes(coarse, medium, fine) -> list[int]:
    """Interleave the 3 SNAC codebooks into Orpheus token indices.

    ``coarse``/``medium``/``fine`` are the per-frame code sequences at the
    SNAC 1/2/4 ratio (lengths T / 2T / 4T). Returns 7 token indices per
    frame in the official Orpheus slot order, each offset by
    ``ORPHEUS_CODE_OFFSET + slot * ORPHEUS_CODEBOOK_SIZE`` so the rendered
    ``<custom_token_N>`` strings map onto the Orpheus tokenizer's audio ids.
    """
    coarse = list(coarse)
    if not coarse:
        raise ValueError("coarse codebook must be non-empty")
    frames = len(coarse)
    coarse = _check_codes(coarse, "coarse", frames)
    medium = _check_codes(list(medium), "medium", 2 * frames)
    fine = _check_codes(list(fine), "fine", 4 * frames)

    size = ORPHEUS_CODEBOOK_SIZE
    base = ORPHEUS_CODE_OFFSET
    out: list[int] = []
    for i in range(frames):
        out.append(coarse[i] + base)
        out.append(medium[2 * i] + base + size)
        out.append(fine[4 * i] + base + 2 * size)
        out.append(fine[4 * i + 1] + base + 3 * size)
        out.append(medium[2 * i + 1] + base + 4 * size)
        out.append(fine[4 * i + 2] + base + 5 * size)
        out.append(fine[4 * i + 3] + base + 6 * size)
    return out


def orpheus_tokens_to_string(indices) -> str:
    """Render Orpheus token indices as a ``<custom_token_N>`` string.

    Indices must be non-bool ints (mirrors :func:`_check_codes` — project
    bool-as-int policy; no float/bool coercion).
    """
    indices = list(indices)
    if not indices:
        raise ValueError("token index list must be non-empty")
    for n in indices:
        if isinstance(n, bool) or not isinstance(n, int):
            raise TypeError("token indices must be non-bool ints")
    return "".join(f"<custom_token_{n}>" for n in indices)


# ---------------------------------------------------------------------------
# Audio loading (lazy soundfile / numpy)
# ---------------------------------------------------------------------------


def load_audio_mono(path: str, *, target_sr: int = SNAC_SAMPLE_RATE):
    """Load an audio file as mono float32 at ``target_sr``.

    Lazy-imports soundfile (friendly ImportError naming the install).
    Off-rate audio is resampled with basic linear interpolation — adequate
    for codec encoding of speech; operators wanting studio-grade resampling
    should resample offline. Returns a 1-D ``np.float32`` array.

    Only a regular file is read: a directory, a FIFO or a device raises
    ``ValueError`` before ``soundfile`` sees it. The check is made before
    ``soundfile`` opens the path, so a path replaced in between is not caught.
    """
    if not isinstance(path, str) or not path:
        raise ValueError("audio path must be a non-empty string")
    if "\x00" in path:
        raise ValueError("audio path must not contain null bytes")
    if is_device_namespace_path(path):
        raise ValueError("audio path must not be a device path")
    if not os.path.exists(path):
        raise FileNotFoundError(
            f"audio file not found: {os.path.basename(path)!r}"
        )
    # Symlink rejection — defence-in-depth; the loader already containment-
    # checks rows under data.audio_dir (v0.17.0 policy).
    mode = os.lstat(path).st_mode
    if stat.S_ISLNK(mode):
        raise ValueError("audio path must not be a symlink")
    # A FIFO would block soundfile, and a device would never end.
    if not stat.S_ISREG(mode):
        raise ValueError("audio path must be a regular file")
    # Byte-size cap before any read — defends against a runaway file.
    if os.path.getsize(path) > _MAX_AUDIO_BYTES:
        raise ValueError(
            f"audio file exceeds the {_MAX_AUDIO_BYTES} byte live-codec cap; "
            "split the clip or pre-encode offline"
        )
    try:
        import soundfile
    except ImportError as exc:  # pragma: no cover - dep present in CI
        raise ImportError(
            "Live-codec TTS needs the 'soundfile' package to read audio. "
            "Install it with `pip install soundfile` (or `pip install "
            "'soup-cli[audio]'`)."
        ) from exc
    import numpy as np

    # Probe metadata FIRST — reject an over-long clip before materialising the
    # whole file into RAM (the post-read check below stays as defence-in-depth).
    info = soundfile.info(path)
    if info.samplerate and info.frames / float(info.samplerate) > _MAX_AUDIO_SECONDS:
        raise ValueError(
            f"audio is {info.frames / float(info.samplerate):.1f} seconds — "
            f"exceeds the {_MAX_AUDIO_SECONDS:.0f} seconds live-codec cap; "
            "split the clip or pre-encode offline"
        )
    # Open via O_NOFOLLOW and hand the fd to soundfile — closes the TOCTOU
    # window between the lstat symlink check above and the read (project
    # v0.65/v0.67 O_NOFOLLOW reader policy). soundfile.read accepts an open
    # file object.
    fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
    try:
        with os.fdopen(fd, "rb") as handle:
            data, sr = soundfile.read(handle, dtype="float32", always_2d=True)
    except OSError as exc:
        # O_NOFOLLOW raises ELOOP if the path became a symlink after the lstat.
        raise ValueError(f"audio path could not be read: {exc}") from exc
    duration = data.shape[0] / float(sr)
    if duration > _MAX_AUDIO_SECONDS:
        raise ValueError(
            f"audio is {duration:.1f} seconds — exceeds the "
            f"{_MAX_AUDIO_SECONDS:.0f} seconds live-codec cap; split the "
            "clip or pre-encode offline"
        )
    mono = data.mean(axis=1)
    if int(sr) != int(target_sr):
        n_out = max(1, int(round(len(mono) * target_sr / float(sr))))
        x_old = np.linspace(0.0, 1.0, num=len(mono), endpoint=False)
        x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
        mono = np.interp(x_new, x_old, mono)
    return mono.astype(np.float32)


# ---------------------------------------------------------------------------
# Orpheus / SNAC encoder
# ---------------------------------------------------------------------------


def _get_snac_model(device: Optional[str] = None):
    """Load (once per device) the SNAC codec model. Lazy snac import."""
    dev = device or "cpu"
    cached = _SNAC_CACHE.get(dev)
    if cached is not None:
        return cached
    try:
        from snac import SNAC
    except ImportError as exc:
        raise ImportError(
            "TTS family 'orpheus' live-codec training requires the 'snac' "
            "package (audio codec). Install it with `pip install snac`, or "
            "pre-encode your audio to codec tokens offline and train with "
            "data.format=chatml."
        ) from exc
    model = SNAC.from_pretrained(SNAC_MODEL_ID).eval().to(dev)
    _SNAC_CACHE[dev] = model
    return model


def encode_audio_orpheus(
    path: str,
    *,
    snac_model: Optional[Any] = None,
    device: Optional[str] = None,
) -> str:
    """Encode one audio file into the Orpheus ``<custom_token_N>`` string.

    ``snac_model`` is injectable (tests / pre-loaded models); when ``None``
    the 24 kHz SNAC checkpoint is loaded once per process per device.
    """
    audio = load_audio_mono(path, target_sr=SNAC_SAMPLE_RATE)
    import torch

    model = snac_model if snac_model is not None else _get_snac_model(device)
    wav = torch.from_numpy(audio).reshape(1, 1, -1)
    if device:
        wav = wav.to(device)
    with torch.no_grad():
        codes = model.encode(wav)
    coarse = [int(c) for c in codes[0][0].tolist()]
    medium = [int(c) for c in codes[1][0].tolist()]
    fine = [int(c) for c in codes[2][0].tolist()]
    return orpheus_tokens_to_string(
        interleave_orpheus_codes(coarse, medium, fine)
    )


def llasa_codes_to_string(codes) -> str:
    """Render XCodec2 ids in Llasa's native speech-token envelope."""
    values = list(codes)
    if not values:
        raise ValueError("Llasa XCodec2 code list must be non-empty")
    rendered = []
    for code in values:
        if isinstance(code, bool) or not isinstance(code, int):
            raise TypeError("Llasa XCodec2 codes must be non-bool ints")
        if not (0 <= code < XCODEC2_CODEBOOK_SIZE):
            raise ValueError(
                f"Llasa XCodec2 code {code} out of range "
                f"[0, {XCODEC2_CODEBOOK_SIZE})"
            )
        rendered.append(f"<|s_{code}|>")
    return (
        "<|SPEECH_GENERATION_START|>"
        + "".join(rendered)
        + "<|SPEECH_GENERATION_END|>"
    )


def _get_xcodec2_components(device: Optional[str] = None):
    """Load Transformers-native XCodec2 lazily at Soup's dependency floor."""
    dev = device or "cpu"
    cached = _XCODEC2_CACHE.get(dev)
    if cached is not None:
        return cached
    try:
        from transformers import AutoFeatureExtractor, Xcodec2Model
    except ImportError as exc:
        raise RuntimeError(
            "Llasa live-codec requires Transformers with native XCodec2 support "
            "(Soup requires transformers>=5.16.1). Upgrade the train extra rather "
            "than installing the legacy xcodec2 package, which pins an old Torch."
        ) from exc
    extractor = AutoFeatureExtractor.from_pretrained(XCODEC2_MODEL_ID)
    model = Xcodec2Model.from_pretrained(XCODEC2_MODEL_ID).eval().to(dev)
    _XCODEC2_CACHE[dev] = (model, extractor)
    return model, extractor


def clear_xcodec2_cache(device: Optional[str] = None) -> None:
    """Release the cached XCodec2 model after dataset encoding (#1112 review)."""
    dev = device or "cpu"
    _XCODEC2_CACHE.pop(dev, None)

    # The live-codec pass runs before the base model/trainer is materialised.
    # Do not keep a ~2.5 GB fp32 XCodec2 checkpoint resident for the entire run.
    import gc

    gc.collect()
    if str(dev).startswith("cuda"):
        try:
            import torch
        except ImportError:  # pragma: no cover - train extra always has torch
            return
        if torch.cuda.is_available():
            torch.cuda.empty_cache()


def encode_audio_llasa(
    path: str,
    *,
    xcodec2_model: Optional[Any] = None,
    feature_extractor: Optional[Any] = None,
    device: Optional[str] = None,
) -> str:
    """Encode 16 kHz speech with Transformers-native XCodec2."""
    audio = load_audio_mono(path, target_sr=XCODEC2_SAMPLE_RATE)
    import torch

    if (xcodec2_model is None) != (feature_extractor is None):
        raise ValueError("xcodec2_model and feature_extractor must be provided together")
    if xcodec2_model is None:
        model, extractor = _get_xcodec2_components(device)
    else:
        model, extractor = xcodec2_model, feature_extractor

    inputs = extractor(
        audio=audio,
        sampling_rate=XCODEC2_SAMPLE_RATE,
        return_tensors="pt",
    )
    if device:
        inputs = {
            key: value.to(device) if hasattr(value, "to") else value
            for key, value in inputs.items()
        }
    with torch.inference_mode():
        encoded = model.encode(**inputs)
    codes = encoded.audio_codes
    values = [int(v) for v in codes.reshape(-1).detach().cpu().tolist()]
    return llasa_codes_to_string(values)


def csm_live_codec_error() -> RuntimeError:
    return RuntimeError(
        "TTS family 'sesame_csm' cannot use Soup's codec-string SFT "
        "live-codec path: CSM trains 32 Mimi codebooks plus text as parallel "
        "multimodal frames, not audio ids embedded in a text assistant turn. "
        "Use Soup's dedicated CSM trainer with training.tts_family=sesame_csm, "
        "data.format=audio and training.quantization=none. Pre-encoded "
        "data.format=chatml is not a native CSM training example."
    )


def incompatible_live_codec_error(family: str) -> RuntimeError:
    """Explain upstream packaging/objective blockers for #265 families."""
    canonical = validate_tts_family(family)
    if canonical == "spark":
        return RuntimeError(
            "TTS family 'spark' has no installable 'sparktts' PyPI package. "
            "The official Spark-TTS source pins torch==2.5.1, torchaudio==2.5.1 "
            "and transformers==4.46.2, which conflicts with Soup's supported "
            "training stack (#265). Refusing raw-audio live encoding instead of "
            "suggesting a nonexistent install or downgrading the environment. "
            "Pre-encode with the official Spark-TTS environment and train with "
            "data.format=chatml until a compatible native codec integration lands."
        )
    if canonical == "oute":
        return RuntimeError(
            "TTS family 'oute' live-codec preparation cannot safely depend on "
            "the current outetts package: its published metadata pins "
            "transformers==4.52.3 while Soup requires transformers>=5.16.1. "
            "Oute v0.3 training also needs transcript/word alignment plus audio "
            "codes, not a bare WAV-to-token substitution (#265). Refusing raw-audio "
            "live encoding rather than silently downgrading Transformers or "
            "training the wrong target. Use an upstream Oute environment to "
            "pre-encode training examples and train with data.format=chatml "
            "until a compatible native path lands."
        )
    return RuntimeError(
        f"TTS family {canonical!r} has no incompatible-live-codec contract"
    )


def tts_encoder_for_family(
    family: str, *, device: Optional[str] = None
) -> Callable[[str], str]:
    """Return the live audio→codec-string encoder for ``family``.

    Only the families in :data:`LIVE_CODEC_FAMILIES` have a codec-string
    encoder. Orpheus uses SNAC and Llasa uses XCodec2. Spark/Oute raw-audio
    encoding is refused for upstream-stack incompatibility; Sesame CSM is
    refused because its parallel Mimi-codebook objective needs a dedicated
    multimodal trainer rather than this text-SFT adapter.
    """
    canonical = validate_tts_family(family)
    if canonical == "orpheus":

        def _encode(path: str) -> str:
            return encode_audio_orpheus(path, device=device)

        return _encode
    if canonical == "llasa":

        def _encode(path: str) -> str:
            return encode_audio_llasa(path, device=device)

        return _encode
    if canonical == "sesame_csm":
        raise csm_live_codec_error()
    if canonical in {"spark", "oute"}:
        raise incompatible_live_codec_error(canonical)
    raise AssertionError(f"unhandled TTS family after validation: {canonical!r}")


# ---------------------------------------------------------------------------
# Dataset mapping
# ---------------------------------------------------------------------------


def encode_tts_row(row: dict, encoder: Callable[[str], str]) -> dict:
    """Encode one ``{"audio", "messages"}`` row into a chat row.

    The codec-token string becomes the FINAL assistant turn: an existing
    trailing assistant message has its content replaced (the audio is the
    training target), otherwise an assistant turn is appended. Returns a NEW
    deep-copied row without the ``audio`` key — the caller's row is never
    mutated.
    """
    import copy

    if not isinstance(row, dict):
        raise TypeError(f"row must be a dict, got {type(row).__name__}")
    audio = row.get("audio")
    if not isinstance(audio, str) or not audio:
        raise ValueError("live-codec TTS row must have a non-empty 'audio' path")
    messages = row.get("messages")
    if not isinstance(messages, (list, tuple)):
        raise ValueError("live-codec TTS row must have a 'messages' list")

    codec_string = encoder(audio)
    new_messages = [copy.deepcopy(m) for m in messages]
    if new_messages and isinstance(new_messages[-1], dict) and (
        new_messages[-1].get("role") == "assistant"
    ):
        new_messages[-1]["content"] = codec_string
    else:
        new_messages.append({"role": "assistant", "content": codec_string})
    out = {
        key: copy.deepcopy(value)
        for key, value in row.items()
        if key not in ("audio", "messages")
    }
    out["messages"] = new_messages
    return out


def encode_tts_dataset(
    dataset: dict,
    family: str,
    *,
    encoder: Optional[Callable[[str], str]] = None,
    device: Optional[str] = None,
    console: Optional[Any] = None,
) -> dict:
    """Encode every ``train`` / ``val`` row's audio into codec-token chat rows.

    Returns a NEW dataset dict (input never mutated). ``encoder`` is
    injectable for tests; by default it is resolved per-family via
    :func:`tts_encoder_for_family` (which raises for not-yet-implemented
    families — #265).
    """
    if not isinstance(dataset, dict):
        raise TypeError(
            f"dataset must be a dict, got {type(dataset).__name__}"
        )
    canonical = validate_tts_family(family)
    encode = encoder if encoder is not None else tts_encoder_for_family(
        canonical, device=device
    )

    new_dataset = dict(dataset)
    total = 0
    for split in ("train", "val"):
        rows = new_dataset.get(split)
        if not isinstance(rows, (list, tuple)):
            continue
        new_dataset[split] = [encode_tts_row(row, encode) for row in rows]
        total += len(rows)
    if console is not None:
        console.print(
            f"[green]TTS live-codec:[/] encoded {total} row(s) "
            f"(family={canonical})"
        )
    return new_dataset