SOTA-Class TTS at Compact Scale
Python
#!/usr/bin/env python3
"""Supervised fine-tuning for audio8_tts 0.6B and 0.1B models."""
from __future__ import annotations
import json
import logging
import math
from dataclasses import dataclass, field
from functools import partial
from pathlib import Path
import torch
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint
from torch.utils.data import Dataset
from transformers import AutoModel, AutoProcessor, HfArgumentParser, Trainer, TrainingArguments
from transformers.trainer_utils import get_last_checkpoint
from audio8_tts_data import (
build_sft_example,
clean_text,
collate_sft_examples,
load_codes,
read_jsonl,
resolve_manifest_path,
validate_sample_id,
)
LOGGER = logging.getLogger(__name__)
PROJECT_ROOT = Path(__file__).resolve().parent
DEFAULT_MODEL = PROJECT_ROOT / "model" / "audio8_tts_0_6B_preview"
@dataclass
class ModelArguments:
model_name_or_path: str = field(
default=str(DEFAULT_MODEL), metadata={"help": "Local checkpoint or Hugging Face model ID."}
)
max_length: int = field(default=2048)
freeze_slow_ar: bool = field(default=False)
freeze_fast_ar: bool = field(default=False)
@dataclass
class DataArguments:
train_jsonl: str | None = field(default=None)
eval_jsonl: str | None = field(default=None)
max_train_samples: int | None = field(default=None)
max_eval_samples: int | None = field(default=None)
@dataclass
class Audio8TTSTrainingArguments(TrainingArguments):
output_dir: str = field(default="outputs/audio8_tts_sft")
export_dir: str | None = field(
default=None,
metadata={"help": "Optional clean Hugging Face export in addition to output_dir."},
)
resume_mode: str = field(
default="none",
metadata={"help": "none, auto/latest, or an explicit Trainer checkpoint path."},
)
slow_loss_weight: float = field(default=1.0)
fast_loss_weight: float = field(default=1.0)
@dataclass(frozen=True)
class PreparedRow:
sample_id: str
text: str
target_codes: Path
reference_codes: Path | None
reference_text: str | None
class Audio8TTSDataset(Dataset):
"""Map-style dataset backed by a prepared audio8_tts JSONL manifest."""
def __init__(
self,
manifest: Path,
*,
tokenizer,
config,
max_length: int,
max_samples: int | None,
) -> None:
self.manifest = manifest.resolve()
self.tokenizer = tokenizer
self.config = config
self.max_length = int(max_length)
self.rows = self._load_rows(max_samples)
def _load_rows(self, max_samples: int | None) -> list[PreparedRow]:
rows: list[PreparedRow] = []
seen: set[str] = set()
for line_number, value in read_jsonl(self.manifest):
try:
sample_id = validate_sample_id(
value.get("id"), fallback=f"row_{line_number:06d}"
)
text = clean_text(value.get("text"), field_name="text")
target_codes = resolve_manifest_path(
value.get("target_codes"), self.manifest, field_name="target_codes"
)
reference_value = value.get("reference_codes")
reference_text_value = value.get("reference_text")
if bool(reference_value) != bool(reference_text_value):
raise ValueError(
"reference_codes and reference_text must be provided together"
)
reference_codes = (
resolve_manifest_path(
reference_value, self.manifest, field_name="reference_codes"
)
if reference_value
else None
)
reference_text = (
clean_text(reference_text_value, field_name="reference_text")
if reference_text_value
else None
)
except (TypeError, ValueError, FileNotFoundError) as exc:
raise ValueError(
f"{self.manifest}:{line_number}: invalid prepared row: {exc}"
) from exc
if sample_id in seen:
raise ValueError(
f"{self.manifest}:{line_number}: duplicate id {sample_id!r}"
)
seen.add(sample_id)
rows.append(
PreparedRow(sample_id, text, target_codes, reference_codes, reference_text)
)
if max_samples is not None and len(rows) >= int(max_samples):
break
if not rows:
raise ValueError(f"no prepared rows found in {self.manifest}")
LOGGER.info("Loaded %d examples from %s", len(rows), self.manifest)
return rows
def __len__(self) -> int:
return len(self.rows)
def __getitem__(self, index: int) -> dict[str, torch.Tensor]:
row = self.rows[index]
target_codes = load_codes(
row.target_codes, num_codebooks=int(self.config.num_codebooks)
)
reference_codes = (
load_codes(row.reference_codes, num_codebooks=int(self.config.num_codebooks))
if row.reference_codes is not None
else None
)
try:
return build_sft_example(
tokenizer=self.tokenizer,
config=self.config,
text=row.text,
target_codes=target_codes,
reference_codes=reference_codes,
reference_text=row.reference_text,
max_length=self.max_length,
)
except ValueError as exc:
raise ValueError(f"failed to encode sample {row.sample_id!r}: {exc}") from exc
def actual_model(model: torch.nn.Module) -> torch.nn.Module:
return model.module if hasattr(model, "module") else model
def set_trainable_modules(model: torch.nn.Module, *, freeze_slow: bool, freeze_fast: bool) -> None:
"""Freeze complete AR branches while preserving the requested branch."""
slow_prefixes = (
"embeddings.",
"codebook_embeddings.",
"layers.",
"norm.",
"slow.",
"semantic_output.",
)
fast_prefixes = (
"fast_project_in.",
"fast_embeddings.",
"fast_layers.",
"fast_norm.",
"fast_output.",
)
for name, parameter in model.named_parameters():
if freeze_slow and name.startswith(slow_prefixes):
parameter.requires_grad = False
if freeze_fast and name.startswith(fast_prefixes):
parameter.requires_grad = False
trainable = sum(
parameter.numel() for parameter in model.parameters() if parameter.requires_grad
)
total = sum(parameter.numel() for parameter in model.parameters())
if trainable == 0:
raise ValueError("freeze_slow_ar and freeze_fast_ar leave no trainable parameters")
LOGGER.info("Trainable parameters: %s / %s (%.2f%%)", trainable, total, 100 * trainable / total)
def slow_loss_targets(labels: torch.Tensor, config, num_classes: int) -> torch.Tensor:
"""Adapt global token IDs to the model's slow output space, without changing inputs."""
if num_classes == int(config.vocab_size):
# The 0.6B head predicts the complete vocabulary, including EOS/newline.
return labels
if num_classes != int(config.codebook_size) + 1:
raise ValueError(
f"Unsupported slow output size {num_classes}: expected full vocabulary "
f"({config.vocab_size}) or semantic tokens plus EOS ({config.codebook_size + 1})"
)
# The 0.1B head predicts only codec semantics and EOS. Keep the original
# labels for semantic frame selection and teacher forcing of the fast AR.
targets = torch.full_like(labels, -100)
semantic = labels.ge(int(config.semantic_begin_id)) & labels.le(int(config.semantic_end_id))
targets[semantic] = labels[semantic] - int(config.semantic_begin_id)
targets[labels.eq(int(config.eos_token_id))] = int(config.codebook_size)
return targets
def fast_codebook_logits(
model: torch.nn.Module,
slow_hidden: torch.Tensor,
codebooks: torch.Tensor,
) -> torch.Tensor:
"""Teacher-force the fast AR branch for all codebooks in parallel."""
prefix = codebooks[:, :-1].long()
fast_dtype = model.fast_embeddings.weight.dtype
hidden = model.fast_project_in(slow_hidden.to(dtype=fast_dtype))
hidden = torch.cat((hidden[:, None, :], model.fast_embeddings(prefix)), dim=1)
length = int(hidden.shape[1])
positions = torch.arange(length, device=hidden.device)
attention_mask = torch.ones(
(hidden.shape[0], length), dtype=torch.long, device=hidden.device
)
mask = model._causal_mask(attention_mask, positions, length)
rope = model.fast_freqs_cis[:length]
for layer in model.fast_layers:
if bool(getattr(model, "gradient_checkpointing", False)) and model.training:
hidden = checkpoint(layer, hidden, rope, mask, use_reentrant=False)
else:
hidden = layer(hidden, rope, mask)
return model.fast_output(model.fast_norm(hidden))
class Audio8TTSTrainer(Trainer):
"""Transformers Trainer with the two losses required by DualAR."""
def __init__(self, *args, slow_loss_weight: float, fast_loss_weight: float, **kwargs):
super().__init__(*args, **kwargs)
# Both custom losses are microbatch means; compute_loss does not use
# num_items_in_batch. Let Trainer normalize gradient accumulation.
self.model_accepts_loss_kwargs = False
self.slow_loss_weight = float(slow_loss_weight)
self.fast_loss_weight = float(fast_loss_weight)
self._metric_sums: dict[str, float] = {}
self._metric_count = 0
def _record_metrics(self, values: dict[str, torch.Tensor]) -> None:
self._metric_count += 1
for name, value in values.items():
self._metric_sums[name] = self._metric_sums.get(name, 0.0) + float(
value.detach().float().cpu()
)
def log(self, logs: dict[str, float], start_time: float | None = None) -> None:
if self._metric_count:
for name, total in self._metric_sums.items():
logs.setdefault(name, total / self._metric_count)
self._metric_sums.clear()
self._metric_count = 0
super().log(logs, start_time=start_time)
def compute_loss(
self,
model,
inputs: dict[str, torch.Tensor],
return_outputs: bool = False,
num_items_in_batch: torch.Tensor | None = None,
):
del num_items_in_batch
labels = inputs.pop("labels")
outputs = model(**inputs, return_dict=True)
core = actual_model(model)
slow_targets = slow_loss_targets(labels[:, 0], core.config, outputs.logits.shape[-1])
slow_loss = F.cross_entropy(
outputs.logits.reshape(-1, outputs.logits.shape[-1]),
slow_targets.reshape(-1),
ignore_index=-100,
)
semantic_mask = labels[:, 0].ge(int(core.config.semantic_begin_id)) & labels[
:, 0
].le(int(core.config.semantic_end_id))
if not semantic_mask.any():
raise ValueError("batch contains no supervised semantic frames")
slow_hidden = outputs.hidden_states[semantic_mask]
if bool(core.config.norm_fastlayer_input):
slow_hidden = core.norm(slow_hidden)
codebooks = labels[:, 1:].permute(0, 2, 1)[semantic_mask]
codebook_logits = fast_codebook_logits(core, slow_hidden, codebooks)
fast_loss = F.cross_entropy(
codebook_logits.reshape(-1, codebook_logits.shape[-1]),
codebooks.reshape(-1),
)
loss = self.slow_loss_weight * slow_loss + self.fast_loss_weight * fast_loss
if not torch.isfinite(loss):
raise FloatingPointError(
f"non-finite loss: slow={float(slow_loss.detach())}, "
f"fast={float(fast_loss.detach())}"
)
with torch.no_grad():
supervised = slow_targets.ne(-100)
slow_labels = slow_targets[supervised]
slow_predictions = outputs.logits[supervised].argmax(dim=-1)
slow_accuracy = slow_predictions.eq(slow_labels).float().mean()
fast_accuracy = codebook_logits.argmax(dim=-1).eq(codebooks).float().mean()
self._record_metrics(
{
"slow_loss": slow_loss,
"fast_loss": fast_loss,
"slow_accuracy": slow_accuracy,
"fast_accuracy": fast_accuracy,
"semantic_frames": loss.new_tensor(float(semantic_mask.sum())),
}
)
if return_outputs:
return loss, {
"logits": outputs.logits,
"codebook_logits": codebook_logits,
}
return loss
def resolve_resume_checkpoint(output_dir: str, mode: str) -> str | None:
value = str(mode or "none").strip()
if value.lower() in {"", "none", "false", "no"}:
return None
if value.lower() in {"auto", "latest"}:
checkpoint_path = get_last_checkpoint(output_dir)
if checkpoint_path is None:
LOGGER.info("No checkpoint found in %s; starting a new run", output_dir)
return checkpoint_path
path = Path(value).expanduser().resolve()
if not path.is_dir():
raise FileNotFoundError(f"resume checkpoint does not exist: {path}")
return str(path)
def sanitize_config_for_save(config) -> None:
"""Remove runtime-only config values that cannot be serialized to JSON."""
for key, value in list(vars(config).items()):
try:
json.dumps(value)
except TypeError:
LOGGER.warning("Dropping non-JSON config field %s (%s)", key, type(value).__name__)
delattr(config, key)
def main() -> None:
parser = HfArgumentParser((ModelArguments, DataArguments, Audio8TTSTrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
logging.basicConfig(
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
level=logging.INFO,
)
if not data_args.train_jsonl:
raise ValueError("--train_jsonl is required")
if model_args.max_length < 2:
raise ValueError("--max_length must be at least 2")
if training_args.slow_loss_weight < 0 or training_args.fast_loss_weight < 0:
raise ValueError("loss weights must be non-negative")
if math.isclose(training_args.slow_loss_weight + training_args.fast_loss_weight, 0.0):
raise ValueError("at least one loss weight must be positive")
training_args.remove_unused_columns = False
processor = AutoProcessor.from_pretrained(
model_args.model_name_or_path, trust_remote_code=True
)
dtype = torch.bfloat16 if training_args.bf16 else (
torch.float16 if training_args.fp16 else torch.float32
)
model = AutoModel.from_pretrained(
model_args.model_name_or_path,
trust_remote_code=True,
dtype=dtype,
)
if int(model_args.max_length) > int(model.config.max_seq_len):
raise ValueError(
f"--max_length={model_args.max_length} exceeds the model context "
f"limit {model.config.max_seq_len}"
)
if training_args.gradient_checkpointing and not model.supports_gradient_checkpointing:
raise ValueError(
"This model does not support gradient checkpointing. "
"Set GRADIENT_CHECKPOINTING=false or pass --gradient_checkpointing false."
)
model.config.use_gradient_checkpointing = bool(training_args.gradient_checkpointing)
set_trainable_modules(
model,
freeze_slow=model_args.freeze_slow_ar,
freeze_fast=model_args.freeze_fast_ar,
)
train_dataset = Audio8TTSDataset(
Path(data_args.train_jsonl),
tokenizer=processor.tokenizer,
config=model.config,
max_length=model_args.max_length,
max_samples=data_args.max_train_samples,
)
eval_dataset = (
Audio8TTSDataset(
Path(data_args.eval_jsonl),
tokenizer=processor.tokenizer,
config=model.config,
max_length=model_args.max_length,
max_samples=data_args.max_eval_samples,
)
if data_args.eval_jsonl
else None
)
collator = partial(
collate_sft_examples, pad_token_id=int(model.config.pad_token_id)
)
trainer = Audio8TTSTrainer(
model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
data_collator=collator,
slow_loss_weight=training_args.slow_loss_weight,
fast_loss_weight=training_args.fast_loss_weight,
)
resume = resolve_resume_checkpoint(training_args.output_dir, training_args.resume_mode)
if training_args.do_train:
result = trainer.train(resume_from_checkpoint=resume)
trainer.log_metrics("train", result.metrics)
trainer.save_metrics("train", result.metrics)
trainer.save_state()
if training_args.do_eval:
if eval_dataset is None:
raise ValueError("--do_eval requires --eval_jsonl")
metrics = trainer.evaluate()
trainer.log_metrics("eval", metrics)
trainer.save_metrics("eval", metrics)
final_model = actual_model(trainer.model)
final_model.gradient_checkpointing_disable()
final_model.config.use_gradient_checkpointing = False
sanitize_config_for_save(final_model.config)
trainer.save_model(training_args.output_dir)
processor.save_pretrained(training_args.output_dir)
if training_args.export_dir:
trainer.save_model(training_args.export_dir)
processor.save_pretrained(training_args.export_dir)
LOGGER.info("Saved final checkpoint to %s", training_args.output_dir)
if __name__ == "__main__":
main()