Skip to content

SOTA-Class TTS at Compact Scale

Python

Fix SFT label contracts and training defaults

chuarunyuan5b43938e6ae1
diff --git a/.gitignore b/.gitignore
index f89a742..aeb3e78 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,7 +20,8 @@ prepared_data/
# Model weights are distributed separately from the source repository.
/model/
-/configs/
+/configs/*
+!/configs/deepspeed_zero2.json
*.safetensors
*.pth
*.pt
diff --git a/audio8_tts_sft.py b/audio8_tts_sft.py
index 028ef12..024db1b 100644
--- a/audio8_tts_sft.py
+++ b/audio8_tts_sft.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Supervised fine-tuning for audio8_tts Preview."""
+"""Supervised fine-tuning for audio8_tts 0.6B and 0.1B models."""
from __future__ import annotations
@@ -9,12 +9,11 @@ import math
from dataclasses import dataclass, field
from functools import partial
from pathlib import Path
-from typing import Any
import torch
import torch.nn.functional as F
-from torch.utils.data import Dataset
from torch.utils.checkpoint import checkpoint
+from torch.utils.data import Dataset
from transformers import AutoModel, AutoProcessor, HfArgumentParser, Trainer, TrainingArguments
from transformers.trainer_utils import get_last_checkpoint
@@ -28,7 +27,6 @@ from audio8_tts_data import (
validate_sample_id,
)
-
LOGGER = logging.getLogger(__name__)
PROJECT_ROOT = Path(__file__).resolve().parent
DEFAULT_MODEL = PROJECT_ROOT / "model" / "audio8_tts_0_6B_preview"
@@ -176,7 +174,14 @@ def actual_model(model: torch.nn.Module) -> torch.nn.Module:
def set_trainable_modules(model: torch.nn.Module, *, freeze_slow: bool, freeze_fast: bool) -> None:
"""Freeze complete AR branches while preserving the requested branch."""
- slow_prefixes = ("embeddings.", "codebook_embeddings.", "layers.", "norm.")
+ slow_prefixes = (
+ "embeddings.",
+ "codebook_embeddings.",
+ "layers.",
+ "norm.",
+ "slow.",
+ "semantic_output.",
+ )
fast_prefixes = (
"fast_project_in.",
"fast_embeddings.",
@@ -189,13 +194,35 @@ def set_trainable_modules(model: torch.nn.Module, *, freeze_slow: bool, freeze_f
parameter.requires_grad = False
if freeze_fast and name.startswith(fast_prefixes):
parameter.requires_grad = False
- trainable = sum(parameter.numel() for parameter in model.parameters() if parameter.requires_grad)
+ trainable = sum(
+ parameter.numel() for parameter in model.parameters() if parameter.requires_grad
+ )
total = sum(parameter.numel() for parameter in model.parameters())
if trainable == 0:
raise ValueError("freeze_slow_ar and freeze_fast_ar leave no trainable parameters")
LOGGER.info("Trainable parameters: %s / %s (%.2f%%)", trainable, total, 100 * trainable / total)
+def slow_loss_targets(labels: torch.Tensor, config, num_classes: int) -> torch.Tensor:
+ """Adapt global token IDs to the model's slow output space, without changing inputs."""
+ if num_classes == int(config.vocab_size):
+ # The 0.6B head predicts the complete vocabulary, including EOS/newline.
+ return labels
+ if num_classes != int(config.codebook_size) + 1:
+ raise ValueError(
+ f"Unsupported slow output size {num_classes}: expected full vocabulary "
+ f"({config.vocab_size}) or semantic tokens plus EOS ({config.codebook_size + 1})"
+ )
+
+ # The 0.1B head predicts only codec semantics and EOS. Keep the original
+ # labels for semantic frame selection and teacher forcing of the fast AR.
+ targets = torch.full_like(labels, -100)
+ semantic = labels.ge(int(config.semantic_begin_id)) & labels.le(int(config.semantic_end_id))
+ targets[semantic] = labels[semantic] - int(config.semantic_begin_id)
+ targets[labels.eq(int(config.eos_token_id))] = int(config.codebook_size)
+ return targets
+
+
def fast_codebook_logits(
model: torch.nn.Module,
slow_hidden: torch.Tensor,
@@ -226,6 +253,9 @@ class Audio8TTSTrainer(Trainer):
def __init__(self, *args, slow_loss_weight: float, fast_loss_weight: float, **kwargs):
super().__init__(*args, **kwargs)
+ # Both custom losses are microbatch means; compute_loss does not use
+ # num_items_in_batch. Let Trainer normalize gradient accumulation.
+ self.model_accepts_loss_kwargs = False
self.slow_loss_weight = float(slow_loss_weight)
self.fast_loss_weight = float(fast_loss_weight)
self._metric_sums: dict[str, float] = {}
@@ -258,9 +288,10 @@ class Audio8TTSTrainer(Trainer):
outputs = model(**inputs, return_dict=True)
core = actual_model(model)
+ slow_targets = slow_loss_targets(labels[:, 0], core.config, outputs.logits.shape[-1])
slow_loss = F.cross_entropy(
outputs.logits.reshape(-1, outputs.logits.shape[-1]),
- labels[:, 0].reshape(-1),
+ slow_targets.reshape(-1),
ignore_index=-100,
)
semantic_mask = labels[:, 0].ge(int(core.config.semantic_begin_id)) & labels[
@@ -286,8 +317,9 @@ class Audio8TTSTrainer(Trainer):
)
with torch.no_grad():
- slow_labels = labels[:, 0][labels[:, 0].ne(-100)]
- slow_predictions = outputs.logits[labels[:, 0].ne(-100)].argmax(dim=-1)
+ supervised = slow_targets.ne(-100)
+ slow_labels = slow_targets[supervised]
+ slow_predictions = outputs.logits[supervised].argmax(dim=-1)
slow_accuracy = slow_predictions.eq(slow_labels).float().mean()
fast_accuracy = codebook_logits.argmax(dim=-1).eq(codebooks).float().mean()
self._record_metrics(
@@ -365,6 +397,11 @@ def main() -> None:
f"--max_length={model_args.max_length} exceeds the model context "
f"limit {model.config.max_seq_len}"
)
+ if training_args.gradient_checkpointing and not model.supports_gradient_checkpointing:
+ raise ValueError(
+ "This model does not support gradient checkpointing. "
+ "Set GRADIENT_CHECKPOINTING=false or pass --gradient_checkpointing false."
+ )
model.config.use_gradient_checkpointing = bool(training_args.gradient_checkpointing)
set_trainable_modules(
model,
diff --git a/audio8_tts_sft.sh b/audio8_tts_sft.sh
index 3ace35b..1785334 100644
--- a/audio8_tts_sft.sh
+++ b/audio8_tts_sft.sh
@@ -21,6 +21,35 @@ if [[ -z "${TRAIN_JSONL}" ]]; then
exit 2
fi
+# Appended Transformers arguments override the environment, as before. Validate
+# the effective path so an explicit --deepspeed can replace a missing default.
+deepspeed_config="${DEEPSPEED_CONFIG:-${PROJECT_ROOT}/configs/deepspeed_zero2.json}"
+expect_deepspeed_config=false
+for arg in "$@"; do
+ if [[ "${expect_deepspeed_config}" == true ]]; then
+ if [[ -z "${arg}" || "${arg}" == --* ]]; then
+ echo "--deepspeed requires a JSON config file path." >&2
+ exit 2
+ fi
+ deepspeed_config="${arg}"
+ expect_deepspeed_config=false
+ else
+ case "${arg}" in
+ --deepspeed) expect_deepspeed_config=true ;;
+ --deepspeed=*) deepspeed_config="${arg#--deepspeed=}" ;;
+ esac
+ fi
+done
+if [[ "${expect_deepspeed_config}" == true ]]; then
+ echo "--deepspeed requires a JSON config file path." >&2
+ exit 2
+fi
+if [[ ! -f "${deepspeed_config}" || ! -r "${deepspeed_config}" ]]; then
+ echo "DeepSpeed config is not a readable file: ${deepspeed_config}" >&2
+ echo "Set DEEPSPEED_CONFIG or append --deepspeed with a valid JSON config path." >&2
+ exit 2
+fi
+
export TOKENIZERS_PARALLELISM="${TOKENIZERS_PARALLELISM:-false}"
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}"
@@ -57,12 +86,12 @@ fi
--save_total_limit "${SAVE_TOTAL_LIMIT:-3}" \
--dataloader_num_workers "${DATALOADER_NUM_WORKERS:-0}" \
--bf16 "${BF16:-true}" \
- --gradient_checkpointing "${GRADIENT_CHECKPOINTING:-true}" \
+ --gradient_checkpointing "${GRADIENT_CHECKPOINTING:-false}" \
--freeze_slow_ar "${FREEZE_SLOW_AR:-false}" \
--freeze_fast_ar "${FREEZE_FAST_AR:-false}" \
--resume_mode "${RESUME_MODE:-none}" \
--report_to "${REPORT_TO:-tensorboard}" \
--remove_unused_columns false \
- --deepspeed "${DEEPSPEED_CONFIG:-${PROJECT_ROOT}/configs/deepspeed_zero2.json}" \
+ --deepspeed "${deepspeed_config}" \
"${training_args[@]}" \
"$@"
diff --git a/configs/deepspeed_zero2.json b/configs/deepspeed_zero2.json
new file mode 100644
index 0000000..17e90e3
--- /dev/null
+++ b/configs/deepspeed_zero2.json
@@ -0,0 +1,16 @@
+{
+ "zero_optimization": {
+ "stage": 2,
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "overlap_comm": true,
+ "reduce_scatter": true,
+ "reduce_bucket_size": 5e8,
+ "contiguous_gradients": true
+ },
+ "gradient_accumulation_steps": "auto",
+ "gradient_clipping": "auto",
+ "train_batch_size": "auto",
+ "train_micro_batch_size_per_gpu": "auto",
+ "bf16": {"enabled": "auto"}
+}
diff --git a/guides/training.md b/guides/training.md
index e0b4559..d6d785b 100644
--- a/guides/training.md
+++ b/guides/training.md
@@ -2,19 +2,35 @@
[All releases](../README.md) · [中文](training_zh.md) · [Inference](inference.md)
-These instructions retain the 0.6B SFT workflow. Run from the repository root and explicitly choose the release to fine-tune.
+Run from the repository root and explicitly choose the release to fine-tune.
```bash
export MODEL=Edge0/Audio8-TTS-Realistic-0.6b
# Preview 0.6B: export MODEL=Edge0/Audio8-TTS-Preview-0.6b
+# Preview 0.1B: export MODEL=Edge0/Audio8-TTS-Preview-0.1b
```
+The trainer adapts the slow loss to the selected model's output head:
+
+| Model | Slow loss targets | Gradient checkpointing |
+| --- | --- | --- |
+| Realistic / Preview 0.6B | Full-vocabulary token IDs, including EOS | Keep disabled; the published model does not support it |
+| Preview 0.1B | Semantic classes 0–4095 and EOS class 4096; other tokens ignored | Disabled by default; may be enabled explicitly |
+
+This mapping applies only to the slow loss and accuracy. Model inputs keep their
+original token IDs, and fast codebook targets remain unchanged.
+
Install the training dependencies first:
```bash
pip install -r requirements-train.txt
```
+For the 0.1B Falcon-H1 backbone, optional `mamba-ssm` and `causal-conv1d`
+packages enable optimized CUDA kernels. Use builds compatible with your
+PyTorch/CUDA environment and check startup warnings for kernel fallbacks.
+These packages are not needed for 0.6B.
+
### 1. Create a raw manifest
The target `audio` field is required. `reference_audio` and `reference_text`
@@ -66,7 +82,29 @@ For multi-node training, set `NNODES`, `NODE_RANK`, `MASTER_ADDR`, and
overridden through the environment variables in `audio8_tts_sft.sh`; additional
Transformers arguments may be appended to the command.
+The launcher includes a [DeepSpeed ZeRO-2 configuration](../configs/deepspeed_zero2.json).
+Its `auto` values follow the training arguments. Set `DEEPSPEED_CONFIG=/path/to/config.json`
+or append `--deepspeed /path/to/config.json` to use another file; an appended
+argument takes precedence. Gradient checkpointing defaults to `false`. For
+0.1B only, set `GRADIENT_CHECKPOINTING=true` if needed to reduce activation memory.
+
+By default, each GPU processes 2 samples per microbatch and accumulates gradients
+over 8 microbatches: 16 samples per GPU per optimizer step, or 128 samples with
+8 GPUs. Step timings should distinguish microsteps from optimizer steps.
+
+### Training objectives and export
+
SFT optimizes both the slow semantic/EOS objective and the fast codebook
-teacher-forcing objective. Set `FREEZE_SLOW_AR=true` or `FREEZE_FAST_AR=true`
-when adapting only one branch. The exported directory remains loadable with
-standard `AutoModel` and `AutoProcessor` APIs using `trust_remote_code=True`.
+teacher-forcing objective. Fast loss covers all 10 codebooks, including the
+first semantic codebook as an auxiliary prediction target. It is retained to
+match the existing training objective.
+
+Each microbatch contributes a mean slow loss plus a mean fast loss (weighted
+by `slow_loss_weight` and `fast_loss_weight`). Gradient accumulation averages
+these microbatch losses; it does not reweight them by token count across the
+accumulation window. Fast accuracy measures exact codebook-token matches and
+is not a direct measure of speech quality.
+
+Set `FREEZE_SLOW_AR=true` or `FREEZE_FAST_AR=true` when adapting only one branch.
+The exported directory remains loadable with standard `AutoModel` and
+`AutoProcessor` APIs using `trust_remote_code=True`.
diff --git a/guides/training_zh.md b/guides/training_zh.md
index a676553..f3895e5 100644
--- a/guides/training_zh.md
+++ b/guides/training_zh.md
@@ -2,19 +2,33 @@
[系列首页](../README_zh.md) · [English](training.md) · [推理](inference_zh.md)
-以下保留 0.6B SFT 流程。从仓库根目录执行,并显式选择需要微调的版本。
+从仓库根目录执行,并显式选择需要微调的版本。
```bash
export MODEL=Edge0/Audio8-TTS-Realistic-0.6b
# Preview 0.6B: export MODEL=Edge0/Audio8-TTS-Preview-0.6b
+# Preview 0.1B: export MODEL=Edge0/Audio8-TTS-Preview-0.1b
```
+Trainer 根据所选模型的输出头适配 slow loss:
+
+| 模型 | Slow loss 目标 | Gradient checkpointing |
+| --- | --- | --- |
+| Realistic / Preview 0.6B | 完整词表中的 token ID,包含 EOS | 保持关闭;已发布模型不支持此功能 |
+| Preview 0.1B | Semantic 类别 0–4095,EOS 类别 4096;其余 token 忽略 | 默认关闭,可显式开启 |
+
+该映射仅用于 slow loss 和 accuracy。模型输入保留原始 token ID,fast codebook 目标保持不变。
+
先安装训练依赖:
```bash
pip install -r requirements-train.txt
```
+0.1B 的 Falcon-H1 主干可通过可选的 `mamba-ssm` 和 `causal-conv1d` 包使用优化后的 CUDA
+kernel。请选择与 PyTorch/CUDA 环境兼容的构建,并查看启动时是否有 kernel 回退警告。
+0.6B 不需要这些包。
+
### 1. 创建原始数据 manifest
目标 `audio` 必填;`reference_audio` 和 `reference_text` 可选,但必须同时出现。
@@ -63,6 +77,23 @@ bash audio8_tts_sft.sh
和输出路径可通过 `audio8_tts_sft.sh` 中列出的环境变量覆盖,也可以在命令末尾附加 Transformers
参数。
-SFT 同时优化 slow semantic/EOS loss 和 fast codebook teacher-forcing loss。只训练其中一个分支时,
-可设置 `FREEZE_SLOW_AR=true` 或 `FREEZE_FAST_AR=true`。导出目录仍可通过标准 `AutoModel` 和
-`AutoProcessor` 接口配合 `trust_remote_code=True` 加载。
+启动脚本附带 [DeepSpeed ZeRO-2 配置](../configs/deepspeed_zero2.json),其中的 `auto` 值跟随训练参数。
+可设置 `DEEPSPEED_CONFIG=/path/to/config.json`,或在命令末尾附加
+`--deepspeed /path/to/config.json` 使用其他配置文件;命令末尾的参数优先。
+Gradient checkpointing 默认关闭。仅对 0.1B,可按需设置 `GRADIENT_CHECKPOINTING=true`
+以减少 activation 显存占用。
+
+默认每张 GPU 的 microbatch 包含 2 条样本,累积 8 个 microbatch 后更新一次参数,即每个
+optimizer step 每张卡处理 16 条样本,8 卡共 128 条。比较耗时前应区分 microstep 和 optimizer step。
+
+### 训练目标与导出
+
+SFT 同时优化 slow semantic/EOS loss 和 fast codebook teacher-forcing loss。Fast loss 保留全部
+10 个码本,包括作为辅助预测目标的第一个 semantic 码本,与已有训练目标保持一致。
+
+每个 microbatch 的 loss 为 slow 平均 loss 与 fast 平均 loss 的加权和,权重分别为
+`slow_loss_weight` 和 `fast_loss_weight`。梯度累积对这些 microbatch loss 取平均,不按整个累积窗口的
+token 总数重新加权。Fast accuracy 衡量码本 token 的精确命中率,并不直接代表语音质量。
+
+只训练其中一个分支时,可设置 `FREEZE_SLOW_AR=true` 或 `FREEZE_FAST_AR=true`。
+导出目录仍可通过标准 `AutoModel` 和 `AutoProcessor` 接口配合 `trust_remote_code=True` 加载。
diff --git a/tests/test_sft.py b/tests/test_sft.py
new file mode 100644
index 0000000..68152e5
--- /dev/null
+++ b/tests/test_sft.py
@@ -0,0 +1,335 @@
+"""CPU regression checks for the two SFT output-label contracts.
+
+These tests use small local modules and a deterministic tokenizer, never model
+downloads or checkpoint weights. They exercise the public data builder and
+Trainer rather than reproducing their implementation.
+"""
+
+import re
+import sys
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn.functional as F
+from torch import nn
+from transformers import TrainingArguments
+
+import audio8_tts_sft as sft
+from audio8_tts_data import build_sft_example, collate_sft_examples
+
+
+def config_for(family: str) -> SimpleNamespace:
+ if family == "0.1b":
+ return SimpleNamespace(
+ vocab_size=69633,
+ codebook_size=4096,
+ num_codebooks=10,
+ semantic_begin_id=65537,
+ semantic_end_id=69632,
+ eos_token_id=228,
+ )
+ return SimpleNamespace(
+ vocab_size=155776,
+ codebook_size=4096,
+ num_codebooks=10,
+ semantic_begin_id=151678,
+ semantic_end_id=155773,
+ eos_token_id=151645,
+ )
+
+
+def tiny_config() -> SimpleNamespace:
+ return SimpleNamespace(
+ vocab_size=128,
+ codebook_size=4,
+ num_codebooks=10,
+ semantic_begin_id=100,
+ semantic_end_id=103,
+ eos_token_id=2,
+ pad_token_id=0,
+ norm_fastlayer_input=False,
+ )
+
+
+def test_compact_targets_map_semantic_and_eos_without_mutating_input():
+ config = config_for("0.1b")
+ labels = torch.tensor([[-100, 65537, 69632, 228, 709, 17]])
+ original = labels.clone()
+
+ targets = sft.slow_loss_targets(labels, config, 4097)
+
+ assert targets.tolist() == [[-100, 0, 4095, 4096, -100, -100]]
+ assert torch.equal(labels, original)
+ assert torch.isfinite(F.cross_entropy(torch.zeros((6, 4097)), targets.reshape(-1)))
+
+
+def test_full_vocabulary_targets_preserve_existing_loss_and_gradients():
+ config = config_for("0.6b")
+ labels = torch.tensor([[-100, 151678, 155773, 151645, 198]])
+ generator = torch.Generator().manual_seed(7)
+ logits_before = torch.randn((5, config.vocab_size), generator=generator).requires_grad_()
+ logits_after = logits_before.detach().clone().requires_grad_()
+
+ targets = sft.slow_loss_targets(labels, config, config.vocab_size)
+ loss_before = F.cross_entropy(logits_before, labels.reshape(-1))
+ loss_after = F.cross_entropy(logits_after, targets.reshape(-1))
+ loss_before.backward()
+ loss_after.backward()
+
+ assert torch.equal(targets, labels)
+ torch.testing.assert_close(loss_after, loss_before, rtol=0, atol=0)
+ torch.testing.assert_close(logits_after.grad, logits_before.grad, rtol=0, atol=0)
+
+
+def test_unknown_slow_output_contract_is_rejected():
+ config = config_for("0.1b")
+ with pytest.raises(ValueError):
+ sft.slow_loss_targets(torch.tensor([[65537, 228]]), config, 4098)
+
+
+class PromptTokenizer:
+ """Small tokenizer that keeps chat delimiters distinct for builder tests."""
+
+ special = {"<|im_start|>": 1, "<|im_end|>": 2, "<|voice|>": 3, "<|speaker:0|>": 4}
+
+ def encode(self, text, add_special_tokens=False):
+ assert add_special_tokens is False
+ pieces = re.findall(r"<\|[^|]+\|>|.", text, flags=re.DOTALL)
+ return [self.special.get(piece, 7 if piece == "\n" else 5) for piece in pieces]
+
+
+def codec_frames(semantic):
+ semantic = torch.tensor(semantic, dtype=torch.long)
+ return torch.stack([(semantic + index) % 4 for index in range(10)])
+
+
+def test_builder_preserves_reference_masking_shift_eos_and_padding():
+ config = tiny_config()
+ target = codec_frames([0, 1, 3])
+ example = build_sft_example(
+ tokenizer=PromptTokenizer(),
+ config=config,
+ text="Hello.",
+ target_codes=target,
+ reference_codes=codec_frames([2, 2]),
+ reference_text="Reference.",
+ )
+ labels = example["labels"]
+ semantic_positions = labels[0].ge(100) & labels[0].le(103)
+
+ # Three target frames are supervised; the two reference frames are inputs
+ # only. The next input column is the target predicted at this position.
+ assert labels[0, semantic_positions].tolist() == [100, 101, 103]
+ assert torch.equal(labels[1:, semantic_positions], target)
+ input_semantic = example["input_ids"][0].ge(100) & example["input_ids"][0].le(103)
+ assert int(input_semantic.sum()) == 5
+ for position in torch.where(semantic_positions)[0].tolist():
+ assert torch.equal(example["input_ids"][:, position + 1], labels[:, position])
+ assert labels[0, labels[0].ne(-100)].tolist() == [100, 101, 103, 2, 7]
+ assert labels[1:, ~semantic_positions].eq(-100).all()
+
+ shorter = build_sft_example(
+ tokenizer=PromptTokenizer(), config=config, text="Hi.", target_codes=codec_frames([1])
+ )
+ batch = collate_sft_examples([example, shorter], pad_token_id=config.pad_token_id)
+ padding = batch["attention_mask"].eq(0)
+ assert padding.any()
+ assert batch["labels"].permute(0, 2, 1)[padding].eq(-100).all()
+ assert batch["input_ids"][:, 0][padding].eq(config.pad_token_id).all()
+ compact = sft.slow_loss_targets(batch["labels"][:, 0], config, 5)
+ assert compact[0, compact[0].ne(-100)].tolist() == [0, 1, 3, 4]
+ assert compact[padding].eq(-100).all()
+
+
+def training_args(path, accumulation=1):
+ return TrainingArguments(
+ output_dir=str(path),
+ use_cpu=True,
+ report_to=[],
+ save_strategy="no",
+ logging_strategy="no",
+ disable_tqdm=True,
+ dataloader_pin_memory=False,
+ gradient_accumulation_steps=accumulation,
+ )
+
+
+class TinyDualAR(nn.Module):
+ """Minimal model exposing the actual slow/fast Trainer interface."""
+
+ def __init__(self, compact):
+ super().__init__()
+ self.config = tiny_config()
+ classes = 5 if compact else 128
+ logits = torch.zeros((1, 6, classes))
+ expected = [0, 0, 3, 4, 1, 0] if compact else [0, 100, 103, 2, 7, 0]
+ for position, prediction in enumerate(expected):
+ logits[0, position, prediction] = 6
+ self.slow_logits = nn.Parameter(logits)
+ self.slow_hidden = nn.Parameter(torch.ones((1, 6, 4)))
+ self.fast_project_in = nn.Identity()
+ self.fast_embeddings = nn.Embedding(4, 4)
+ self.fast_layers = nn.ModuleList()
+ self.fast_norm = nn.Identity()
+ self.fast_output = nn.Linear(4, 4, bias=False)
+ self.register_buffer("fast_freqs_cis", torch.zeros((10, 4)))
+
+ @staticmethod
+ def _causal_mask(attention_mask, positions, length):
+ return torch.ones((length, length), dtype=torch.bool).tril()[None, None]
+
+ def forward(self, input_ids, attention_mask=None, return_dict=True, **kwargs):
+ return SimpleNamespace(logits=self.slow_logits, hidden_states=self.slow_hidden)
+
+
[email protected]("compact", [False, True], ids=["full-vocabulary", "compact"])
+def test_trainer_uses_correct_slow_accuracy_and_keeps_all_fast_targets(
+ tmp_path, monkeypatch, compact
+):
+ model = TinyDualAR(compact)
+ trainer = sft.Audio8TTSTrainer(
+ model=model,
+ args=training_args(tmp_path),
+ slow_loss_weight=1.0,
+ fast_loss_weight=1.0,
+ )
+ labels = torch.full((1, 11, 6), -100, dtype=torch.long)
+ labels[0, 0] = torch.tensor([-100, 100, 103, 2, 7, -100])
+ expected_codebooks = codec_frames([0, 3]).T
+ labels[0, 1:, 1:3] = expected_codebooks.T
+ original = labels.clone()
+ seen = []
+ original_fast_logits = sft.fast_codebook_logits
+
+ def capture_fast_inputs(core, hidden, codebooks):
+ seen.append(codebooks.detach().clone())
+ return original_fast_logits(core, hidden, codebooks)
+
+ monkeypatch.setattr(sft, "fast_codebook_logits", capture_fast_inputs)
+ loss, outputs = trainer.compute_loss(
+ model,
+ {"input_ids": torch.zeros((1, 11, 6), dtype=torch.long), "labels": labels},
+ return_outputs=True,
+ )
+ assert torch.isfinite(loss)
+ assert torch.equal(labels, original)
+ assert len(seen) == 1
+ assert torch.equal(seen[0], expected_codebooks)
+ assert outputs["codebook_logits"].shape == (2, 10, 4)
+ assert trainer._metric_sums["slow_accuracy"] == pytest.approx(1.0)
+ assert trainer._metric_sums["semantic_frames"] == 2
+ expected_fast_loss = F.cross_entropy(
+ outputs["codebook_logits"].reshape(-1, 4), expected_codebooks.reshape(-1)
+ )
+ assert trainer._metric_sums["fast_loss"] == pytest.approx(float(expected_fast_loss.detach()))
+ loss.backward()
+ assert model.slow_logits.grad is not None
+ assert model.slow_hidden.grad is not None
+ assert model.slow_hidden.grad[:, 1:3].abs().sum() > 0
+
+
+def test_main_rejects_unsupported_checkpointing_before_dataset_access(tmp_path, monkeypatch):
+ model = TinyDualAR(compact=False)
+ model.supports_gradient_checkpointing = False
+ model.config.max_seq_len = 2048
+ monkeypatch.setattr(
+ sft.AutoProcessor,
+ "from_pretrained",
+ lambda *args, **kwargs: SimpleNamespace(tokenizer=PromptTokenizer()),
+ )
+ monkeypatch.setattr(sft.AutoModel, "from_pretrained", lambda *args, **kwargs: model)
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "audio8_tts_sft.py",
+ "--model_name_or_path",
+ "local-test-model",
+ "--train_jsonl",
+ str(tmp_path / "does-not-exist.jsonl"),
+ "--output_dir",
+ str(tmp_path / "output"),
+ "--gradient_checkpointing",
+ "true",
+ "--use_cpu",
+ "true",
+ "--report_to",
+ "none",
+ ],
+ )
+ with pytest.raises(ValueError, match="does not support gradient checkpointing"):
+ sft.main()
+
+
+class Branches(nn.Module):
+ def __init__(self, family):
+ super().__init__()
+ if family == "0.1b":
+ self.slow = nn.Linear(2, 2)
+ self.semantic_output = nn.Linear(2, 2)
+ else:
+ self.embeddings = nn.Embedding(4, 2)
+ self.layers = nn.ModuleList([nn.Linear(2, 2)])
+ self.norm = nn.LayerNorm(2)
+ self.codebook_embeddings = nn.Embedding(4, 2)
+ self.fast_project_in = nn.Linear(2, 2)
+ self.fast_embeddings = nn.Embedding(4, 2)
+ self.fast_layers = nn.ModuleList([nn.Linear(2, 2)])
+ self.fast_norm = nn.LayerNorm(2)
+ self.fast_output = nn.Linear(2, 2)
+
+
[email protected]("family", ["0.1b", "0.6b"])
[email protected]("freeze_slow,freeze_fast", [(False, False), (True, False), (False, True)])
+def test_freeze_flags_cover_both_architectures(family, freeze_slow, freeze_fast):
+ model = Branches(family)
+ fast_parameters = {
+ id(parameter) for name, parameter in model.named_parameters() if name.startswith("fast_")
+ }
+ sft.set_trainable_modules(model, freeze_slow=freeze_slow, freeze_fast=freeze_fast)
+ for parameter in model.parameters():
+ should_freeze = freeze_fast if id(parameter) in fast_parameters else freeze_slow
+ assert parameter.requires_grad is not should_freeze
+
+
[email protected]("family", ["0.1b", "0.6b"])
+def test_freezing_every_branch_is_rejected(family):
+ with pytest.raises(ValueError, match="no trainable parameters"):
+ sft.set_trainable_modules(Branches(family), freeze_slow=True, freeze_fast=True)
+
+
+class ScalarMeanModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = nn.Parameter(torch.tensor(1.0))
+
+ def forward(self, values, **kwargs):
+ # **kwargs makes Transformers infer loss-kwargs support unless the
+ # Audio8 Trainer explicitly declares its microbatch-mean interface.
+ return self.weight * values
+
+
+class ScalarMeanTrainer(sft.Audio8TTSTrainer):
+ def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
+ return model(inputs["values"]).square().mean()
+
+
+def test_trainer_mean_loss_accumulation_matches_equal_sized_batch(tmp_path):
+ gradients = []
+ values = torch.tensor([1.0, 2.0, 3.0, 4.0])
+ for accumulation in (1, 4):
+ model = ScalarMeanModel()
+ trainer = ScalarMeanTrainer(
+ model=model,
+ args=training_args(tmp_path / str(accumulation), accumulation),
+ slow_loss_weight=1.0,
+ fast_loss_weight=1.0,
+ )
+ trainer.current_gradient_accumulation_steps = accumulation
+ for microbatch in values.chunk(accumulation):
+ trainer.training_step(model, {"values": microbatch}, num_items_in_batch=torch.tensor(4))
+ gradients.append(model.weight.grad.detach().clone())
+
+ torch.testing.assert_close(gradients[0], torch.tensor(15.0))
+ torch.testing.assert_close(gradients[1], gradients[0])
diff --git a/tests/test_sft_launcher.py b/tests/test_sft_launcher.py
new file mode 100644
index 0000000..01acaa7
--- /dev/null
+++ b/tests/test_sft_launcher.py
@@ -0,0 +1,101 @@
+"""Validate launcher arguments without importing torch or starting training."""
+
+import argparse
+import json
+import os
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def launch(tmp_path):
+ capture = tmp_path / "capture.py"
+ capture.write_text("import json, sys\nprint(json.dumps(sys.argv[1:]))\n")
+ fake_python = tmp_path / "fake python"
+ fake_python.write_text(
+ f'#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(capture))} "$@"\n'
+ )
+ fake_python.chmod(0o755)
+
+ def run(args=(), **overrides):
+ env = {
+ "PATH": os.environ["PATH"],
+ "PYTHON": str(fake_python),
+ "TRAIN_JSONL": "prepared_data/train.jsonl",
+ **overrides,
+ }
+ return subprocess.run(
+ ["bash", str(ROOT / "audio8_tts_sft.sh"), *args],
+ cwd=tmp_path,
+ env=env,
+ text=True,
+ capture_output=True,
+ timeout=10,
+ )
+
+ return run
+
+
+def captured_options(result):
+ assert result.returncode == 0, result.stderr
+ argv = json.loads(result.stdout)
+ assert argv[:2] == ["-m", "torch.distributed.run"]
+ parser = argparse.ArgumentParser(allow_abbrev=False)
+ parser.add_argument("--deepspeed")
+ parser.add_argument("--gradient_checkpointing")
+ options, _ = parser.parse_known_args(argv)
+ return options
+
+
+def test_defaults_use_bundled_zero2_and_disable_checkpointing(launch):
+ options = captured_options(launch())
+ config = ROOT / "configs" / "deepspeed_zero2.json"
+ assert options.deepspeed == str(config)
+ assert json.loads(config.read_text())["zero_optimization"]["stage"] == 2
+ assert options.gradient_checkpointing == "false"
+
+
[email protected]("source", ["env", "separate", "equals"])
+def test_custom_config_overrides_default(launch, tmp_path, source):
+ config = tmp_path / "custom config.json"
+ config.write_text('{"zero_optimization": {"stage": 2}}')
+ env = {"DEEPSPEED_CONFIG": str(tmp_path / "missing.json")}
+ args = []
+ if source == "env":
+ env["DEEPSPEED_CONFIG"] = str(config)
+ elif source == "separate":
+ args = ["--deepspeed", str(config)]
+ else:
+ args = [f"--deepspeed={config}"]
+ assert captured_options(launch(args, **env)).deepspeed == str(config)
+
+
+ "args",
+ [
+ [],
+ ["--deepspeed", "missing.json"],
+ ["--deepspeed"],
+ ["--deepspeed="],
+ ["--deepspeed", "--bf16", "false"],
+ ],
+)
+def test_bad_config_fails_before_invoking_python(launch, tmp_path, args):
+ result = launch(args, DEEPSPEED_CONFIG=str(tmp_path / "missing.json"))
+ assert result.returncode == 2
+ assert result.stdout == ""
+ assert (
+ "DeepSpeed config is not a readable file:" in result.stderr
+ or "--deepspeed requires a JSON config file path." in result.stderr
+ )
+
+
+def test_checkpointing_can_be_explicitly_enabled(launch):
+ options = captured_options(launch(GRADIENT_CHECKPOINTING="true"))
+ assert options.gradient_checkpointing == "true"