Skip to content

SOTA-Class TTS at Compact Scale

Python
14.5 KB
Raw Blame
#!/usr/bin/env python3
"""Command-line inference for audio8_tts Preview."""

from __future__ import annotations

import argparse
import random
from dataclasses import dataclass
from pathlib import Path
from typing import Any

import numpy as np
import soundfile as sf
import torch
from tqdm import tqdm
from transformers import AutoModel, AutoProcessor

from audio8_tts_data import chunks, clean_text, json_line, read_jsonl, resolve_manifest_path
from audio8_tts_data import validate_sample_id


PROJECT_ROOT = Path(__file__).resolve().parent
DEFAULT_MODEL = PROJECT_ROOT / "model" / "audio8_tts_0_6B_preview"


@dataclass(frozen=True)
class InferenceItem:
    sample_id: str
    text: str
    output: Path
    reference_audio: Path | None = None
    reference_text: str | None = None


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Synthesize speech with audio8_tts Preview.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    source = parser.add_mutually_exclusive_group(required=True)
    source.add_argument("--text", help="Text for one utterance.")
    source.add_argument("--input-jsonl", type=Path, help="Batch input manifest.")
    parser.add_argument("--reference-audio", type=Path)
    parser.add_argument("--reference-text")
    parser.add_argument("--output", type=Path, default=Path("output.wav"))
    parser.add_argument("--output-dir", type=Path, default=Path("outputs"))
    parser.add_argument("--manifest", type=Path)
    parser.add_argument("--failures", type=Path)
    parser.add_argument("--model", default=str(DEFAULT_MODEL))
    parser.add_argument("--device", default="auto", help="auto, cpu, cuda, or cuda:N")
    parser.add_argument(
        "--dtype", choices=("auto", "bfloat16", "float16", "float32"), default="auto"
    )
    parser.add_argument("--batch-size", type=int, default=1)
    parser.add_argument("--max-new-tokens", type=int, default=1024)
    parser.add_argument("--retry-max-new-tokens", type=int, default=2000)
    parser.add_argument("--temperature", type=float, default=0.8)
    parser.add_argument("--top-p", type=float, default=0.95)
    parser.add_argument("--top-k", type=int, default=50)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--greedy", action="store_true")
    parser.add_argument("--save-codes", action="store_true")
    parser.add_argument("--overwrite", action="store_true")
    return parser.parse_args()


def resolve_device(value: str) -> torch.device:
    if value == "auto":
        value = "cuda" if torch.cuda.is_available() else "cpu"
    device = torch.device(value)
    if device.type == "cuda" and not torch.cuda.is_available():
        raise RuntimeError("CUDA was requested but is not available")
    return device


def resolve_dtype(value: str, device: torch.device) -> torch.dtype:
    if value == "auto":
        return torch.bfloat16 if device.type == "cuda" else torch.float32
    dtype = getattr(torch, value)
    if device.type == "cpu" and dtype != torch.float32:
        raise ValueError("CPU inference requires --dtype float32")
    return dtype


def validate_args(args: argparse.Namespace) -> None:
    if args.batch_size < 1:
        raise ValueError("--batch-size must be positive")
    if args.max_new_tokens < 1:
        raise ValueError("--max-new-tokens must be positive")
    if args.retry_max_new_tokens < args.max_new_tokens:
        raise ValueError("--retry-max-new-tokens must be >= --max-new-tokens")
    if args.temperature <= 0:
        raise ValueError("--temperature must be positive")
    if not 0 < args.top_p <= 1:
        raise ValueError("--top-p must be in (0, 1]")
    if args.top_k < 0:
        raise ValueError("--top-k must be non-negative")
    if (args.reference_audio is None) != (args.reference_text is None):
        raise ValueError("--reference-audio and --reference-text must be provided together")
    if args.input_jsonl is not None and (
        args.reference_audio is not None or args.reference_text is not None
    ):
        raise ValueError("reference options belong in each batch JSONL row")


def load_batch_items(manifest: Path, output_dir: Path) -> list[InferenceItem]:
    manifest = manifest.resolve()
    items: list[InferenceItem] = []
    seen: set[str] = set()
    for line_number, row in read_jsonl(manifest):
        try:
            sample_id = validate_sample_id(row.get("id"), fallback=f"row_{line_number:06d}")
            text = clean_text(row.get("text"), field_name="text")
            reference_value = row.get("reference_audio")
            reference_text_value = row.get("reference_text")
            if bool(reference_value) != bool(reference_text_value):
                raise ValueError("reference_audio and reference_text must be provided together")
            reference_audio = (
                resolve_manifest_path(reference_value, manifest, field_name="reference_audio")
                if reference_value
                else None
            )
            reference_text = (
                clean_text(reference_text_value, field_name="reference_text")
                if reference_text_value
                else None
            )
        except (KeyError, TypeError, ValueError, FileNotFoundError) as exc:
            raise ValueError(f"{manifest}:{line_number}: invalid inference row: {exc}") from exc
        if sample_id in seen:
            raise ValueError(f"{manifest}:{line_number}: duplicate id {sample_id!r}")
        seen.add(sample_id)
        items.append(
            InferenceItem(
                sample_id=sample_id,
                text=text,
                output=output_dir / f"{sample_id}.wav",
                reference_audio=reference_audio,
                reference_text=reference_text,
            )
        )
    if not items:
        raise ValueError(f"no inference rows found in {manifest}")
    return items


def load_items(args: argparse.Namespace) -> list[InferenceItem]:
    if args.input_jsonl is not None:
        return load_batch_items(args.input_jsonl, args.output_dir)
    reference_audio = args.reference_audio.resolve() if args.reference_audio else None
    if reference_audio is not None and not reference_audio.is_file():
        raise FileNotFoundError(f"reference audio does not exist: {reference_audio}")
    return [
        InferenceItem(
            sample_id=validate_sample_id(args.output.stem, fallback="output"),
            text=clean_text(args.text),
            output=args.output,
            reference_audio=reference_audio,
            reference_text=(
                clean_text(args.reference_text, field_name="reference_text")
                if args.reference_text
                else None
            ),
        )
    ]


def main() -> None:
    args = parse_args()
    validate_args(args)
    device = resolve_device(args.device)
    dtype = resolve_dtype(args.dtype, device)
    items = load_items(args)

    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(args.seed)

    for item in items:
        item.output.parent.mkdir(parents=True, exist_ok=True)
    batch_mode = args.input_jsonl is not None
    manifest_path = args.manifest or (args.output_dir / "manifest.jsonl")
    failures_path = args.failures or (args.output_dir / "failures.jsonl")
    if batch_mode:
        manifest_path.parent.mkdir(parents=True, exist_ok=True)
        failures_path.parent.mkdir(parents=True, exist_ok=True)

    skipped = [
        item
        for item in items
        if not args.overwrite and item.output.is_file() and item.output.stat().st_size > 0
    ]
    pending = [item for item in items if item not in skipped]
    print(
        f"[audio8_tts] model={args.model} device={device} dtype={dtype} "
        f"items={len(items)} pending={len(pending)} skipped={len(skipped)}"
    )

    if not pending:
        if batch_mode:
            with manifest_path.open("w", encoding="utf-8") as handle:
                for item in skipped:
                    handle.write(
                        json_line(
                            {
                                "id": item.sample_id,
                                "status": "SKIP",
                                "output_audio": str(item.output),
                                "reference_audio": (
                                    str(item.reference_audio) if item.reference_audio else None
                                ),
                            }
                        )
                    )
            failures_path.write_text("", encoding="utf-8")
            print(f"[audio8_tts] manifest={manifest_path} failures={failures_path}")
        else:
            print(f"[audio8_tts] output={skipped[0].output} (already exists)")
        return

    processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
    model = AutoModel.from_pretrained(
        args.model, trust_remote_code=True, dtype=dtype
    ).eval().to(device)
    sample_rate = int(model.config.codec_sample_rate)
    generator = torch.Generator(device=device).manual_seed(args.seed)

    def synthesize(batch: list[InferenceItem], max_new_tokens: int) -> dict[str, Any]:
        processor_kwargs: dict[str, Any] = {
            "text": [item.text for item in batch],
            "return_tensors": "pt",
        }
        if batch[0].reference_audio is not None:
            processor_kwargs.update(
                reference_audio=[item.reference_audio for item in batch],
                reference_text=[item.reference_text for item in batch],
            )
        inputs = processor(**processor_kwargs)
        inputs = {name: value.to(device) for name, value in inputs.items()}
        output = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=args.temperature,
            top_p=args.top_p,
            top_k=args.top_k,
            do_sample=not args.greedy,
            generator=generator,
            return_dict_in_generate=True,
        )
        waveforms, waveform_lengths = model.decode_audio(output.codes)
        return {
            "output": output,
            "waveforms": waveforms,
            "waveform_lengths": waveform_lengths,
        }

    def run_batch(batch: list[InferenceItem]) -> list[dict[str, Any]]:
        result = synthesize(batch, args.max_new_tokens)
        output = result["output"]
        records: list[dict[str, Any]] = []
        for index, item in enumerate(batch):
            finished = bool(output.finished[index])
            code_length = int(output.code_lengths[index])
            codes = output.codes[index, :, :code_length]
            waveform_length = int(result["waveform_lengths"][index])
            waveform = result["waveforms"][index, :waveform_length]
            if not finished and args.retry_max_new_tokens > args.max_new_tokens:
                retry = synthesize([item], args.retry_max_new_tokens)
                retry_output = retry["output"]
                finished = bool(retry_output.finished[0])
                code_length = int(retry_output.code_lengths[0])
                codes = retry_output.codes[0, :, :code_length]
                waveform_length = int(retry["waveform_lengths"][0])
                waveform = retry["waveforms"][0, :waveform_length]

            waveform_array = waveform.float().cpu().numpy()
            codes_array = codes.cpu().numpy()
            sf.write(item.output, waveform_array, sample_rate)
            if args.save_codes:
                np.save(item.output.with_suffix(".npy"), codes_array)
            records.append(
                {
                    "id": item.sample_id,
                    "status": "OK" if finished else "NO_EOS",
                    "output_audio": str(item.output),
                    "reference_audio": (
                        str(item.reference_audio) if item.reference_audio is not None else None
                    ),
                    "code_frames": int(codes_array.shape[1]),
                    "waveform_samples": int(waveform_array.shape[0]),
                    "sample_rate": sample_rate,
                }
            )
        return records

    records = [
        {
            "id": item.sample_id,
            "status": "SKIP",
            "output_audio": str(item.output),
            "reference_audio": str(item.reference_audio) if item.reference_audio else None,
        }
        for item in skipped
    ]
    failures: list[dict[str, str]] = []
    # The processor treats reference conditioning as a batch-wide mode, so the
    # two groups must not be mixed in one call.
    groups = [
        [item for item in pending if item.reference_audio is None],
        [item for item in pending if item.reference_audio is not None],
    ]
    total_batches = sum((len(group) + args.batch_size - 1) // args.batch_size for group in groups)
    progress = tqdm(total=total_batches, desc="audio8_tts")
    for group in groups:
        for batch_values in chunks(group, args.batch_size):
            batch = list(batch_values)
            try:
                records.extend(run_batch(batch))
            except Exception as batch_exc:
                print(
                    f"[audio8_tts] batch fallback after {type(batch_exc).__name__}: {batch_exc}",
                    flush=True,
                )
                if device.type == "cuda":
                    torch.cuda.empty_cache()
                for item in batch:
                    try:
                        records.extend(run_batch([item]))
                    except Exception as exc:
                        failures.append(
                            {
                                "id": item.sample_id,
                                "error_type": type(exc).__name__,
                                "error": str(exc),
                            }
                        )
                        print(
                            f"[audio8_tts] FAILED {item.sample_id}: "
                            f"{type(exc).__name__}: {exc}",
                            flush=True,
                        )
            progress.update(1)
    progress.close()

    if batch_mode:
        with manifest_path.open("w", encoding="utf-8") as handle:
            handle.writelines(json_line(record) for record in records)
        with failures_path.open("w", encoding="utf-8") as handle:
            handle.writelines(json_line(record) for record in failures)
        print(f"[audio8_tts] manifest={manifest_path} failures={failures_path}")
    else:
        if records:
            print(f"[audio8_tts] output={records[-1]['output_audio']}")
    if failures:
        raise RuntimeError(f"inference failed for {len(failures)} sample(s)")


if __name__ == "__main__":
    main()