SOTA-Class TTS at Compact Scale
Python
Fix SFT label contracts and training defaults
chuarunyuan5b43938e6ae1
diff --git a/.gitignore b/.gitignoreindex f89a742..aeb3e78 100644--- a/.gitignore+++ b/.gitignore@@ -20,7 +20,8 @@ prepared_data/# Model weights are distributed separately from the source repository./model/-/configs/+/configs/*+!/configs/deepspeed_zero2.json*.safetensors*.pth*.ptdiff --git a/audio8_tts_sft.py b/audio8_tts_sft.pyindex 028ef12..024db1b 100644--- a/audio8_tts_sft.py+++ b/audio8_tts_sft.py@@ -1,5 +1,5 @@#!/usr/bin/env python3-"""Supervised fine-tuning for audio8_tts Preview."""+"""Supervised fine-tuning for audio8_tts 0.6B and 0.1B models."""from __future__ import annotations@@ -9,12 +9,11 @@ import mathfrom dataclasses import dataclass, fieldfrom functools import partialfrom pathlib import Path-from typing import Anyimport torchimport torch.nn.functional as F-from torch.utils.data import Datasetfrom torch.utils.checkpoint import checkpoint+from torch.utils.data import Datasetfrom transformers import AutoModel, AutoProcessor, HfArgumentParser, Trainer, TrainingArgumentsfrom transformers.trainer_utils import get_last_checkpoint@@ -28,7 +27,6 @@ from audio8_tts_data import (validate_sample_id,)-LOGGER = logging.getLogger(__name__)PROJECT_ROOT = Path(__file__).resolve().parentDEFAULT_MODEL = PROJECT_ROOT / "model" / "audio8_tts_0_6B_preview"@@ -176,7 +174,14 @@ def actual_model(model: torch.nn.Module) -> torch.nn.Module:def set_trainable_modules(model: torch.nn.Module, *, freeze_slow: bool, freeze_fast: bool) -> None:"""Freeze complete AR branches while preserving the requested branch."""- slow_prefixes = ("embeddings.", "codebook_embeddings.", "layers.", "norm.")+ slow_prefixes = (+ "embeddings.",+ "codebook_embeddings.",+ "layers.",+ "norm.",+ "slow.",+ "semantic_output.",+ )fast_prefixes = ("fast_project_in.","fast_embeddings.",@@ -189,13 +194,35 @@ def set_trainable_modules(model: torch.nn.Module, *, freeze_slow: bool, freeze_fparameter.requires_grad = Falseif freeze_fast and name.startswith(fast_prefixes):parameter.requires_grad = False- trainable = sum(parameter.numel() for parameter in model.parameters() if parameter.requires_grad)+ trainable = sum(+ parameter.numel() for parameter in model.parameters() if parameter.requires_grad+ )total = sum(parameter.numel() for parameter in model.parameters())if trainable == 0:raise ValueError("freeze_slow_ar and freeze_fast_ar leave no trainable parameters")LOGGER.info("Trainable parameters: %s / %s (%.2f%%)", trainable, total, 100 * trainable / total)+def slow_loss_targets(labels: torch.Tensor, config, num_classes: int) -> torch.Tensor:+ """Adapt global token IDs to the model's slow output space, without changing inputs."""+ if num_classes == int(config.vocab_size):+ # The 0.6B head predicts the complete vocabulary, including EOS/newline.+ return labels+ if num_classes != int(config.codebook_size) + 1:+ raise ValueError(+ f"Unsupported slow output size {num_classes}: expected full vocabulary "+ f"({config.vocab_size}) or semantic tokens plus EOS ({config.codebook_size + 1})"+ )++ # The 0.1B head predicts only codec semantics and EOS. Keep the original+ # labels for semantic frame selection and teacher forcing of the fast AR.+ targets = torch.full_like(labels, -100)+ semantic = labels.ge(int(config.semantic_begin_id)) & labels.le(int(config.semantic_end_id))+ targets[semantic] = labels[semantic] - int(config.semantic_begin_id)+ targets[labels.eq(int(config.eos_token_id))] = int(config.codebook_size)+ return targets++def fast_codebook_logits(model: torch.nn.Module,slow_hidden: torch.Tensor,@@ -226,6 +253,9 @@ class Audio8TTSTrainer(Trainer):def __init__(self, *args, slow_loss_weight: float, fast_loss_weight: float, **kwargs):super().__init__(*args, **kwargs)+ # Both custom losses are microbatch means; compute_loss does not use+ # num_items_in_batch. Let Trainer normalize gradient accumulation.+ self.model_accepts_loss_kwargs = Falseself.slow_loss_weight = float(slow_loss_weight)self.fast_loss_weight = float(fast_loss_weight)self._metric_sums: dict[str, float] = {}@@ -258,9 +288,10 @@ class Audio8TTSTrainer(Trainer):outputs = model(**inputs, return_dict=True)core = actual_model(model)+ slow_targets = slow_loss_targets(labels[:, 0], core.config, outputs.logits.shape[-1])slow_loss = F.cross_entropy(outputs.logits.reshape(-1, outputs.logits.shape[-1]),- labels[:, 0].reshape(-1),+ slow_targets.reshape(-1),ignore_index=-100,)semantic_mask = labels[:, 0].ge(int(core.config.semantic_begin_id)) & labels[@@ -286,8 +317,9 @@ class Audio8TTSTrainer(Trainer):)with torch.no_grad():- slow_labels = labels[:, 0][labels[:, 0].ne(-100)]- slow_predictions = outputs.logits[labels[:, 0].ne(-100)].argmax(dim=-1)+ supervised = slow_targets.ne(-100)+ slow_labels = slow_targets[supervised]+ slow_predictions = outputs.logits[supervised].argmax(dim=-1)slow_accuracy = slow_predictions.eq(slow_labels).float().mean()fast_accuracy = codebook_logits.argmax(dim=-1).eq(codebooks).float().mean()self._record_metrics(@@ -365,6 +397,11 @@ def main() -> None:f"--max_length={model_args.max_length} exceeds the model context "f"limit {model.config.max_seq_len}")+ if training_args.gradient_checkpointing and not model.supports_gradient_checkpointing:+ raise ValueError(+ "This model does not support gradient checkpointing. "+ "Set GRADIENT_CHECKPOINTING=false or pass --gradient_checkpointing false."+ )model.config.use_gradient_checkpointing = bool(training_args.gradient_checkpointing)set_trainable_modules(model,diff --git a/audio8_tts_sft.sh b/audio8_tts_sft.shindex 3ace35b..1785334 100644--- a/audio8_tts_sft.sh+++ b/audio8_tts_sft.sh@@ -21,6 +21,35 @@ if [[ -z "${TRAIN_JSONL}" ]]; thenexit 2fi+# Appended Transformers arguments override the environment, as before. Validate+# the effective path so an explicit --deepspeed can replace a missing default.+deepspeed_config="${DEEPSPEED_CONFIG:-${PROJECT_ROOT}/configs/deepspeed_zero2.json}"+expect_deepspeed_config=false+for arg in "$@"; do+ if [[ "${expect_deepspeed_config}" == true ]]; then+ if [[ -z "${arg}" || "${arg}" == --* ]]; then+ echo "--deepspeed requires a JSON config file path." >&2+ exit 2+ fi+ deepspeed_config="${arg}"+ expect_deepspeed_config=false+ else+ case "${arg}" in+ --deepspeed) expect_deepspeed_config=true ;;+ --deepspeed=*) deepspeed_config="${arg#--deepspeed=}" ;;+ esac+ fi+done+if [[ "${expect_deepspeed_config}" == true ]]; then+ echo "--deepspeed requires a JSON config file path." >&2+ exit 2+fi+if [[ ! -f "${deepspeed_config}" || ! -r "${deepspeed_config}" ]]; then+ echo "DeepSpeed config is not a readable file: ${deepspeed_config}" >&2+ echo "Set DEEPSPEED_CONFIG or append --deepspeed with a valid JSON config path." >&2+ exit 2+fi+export TOKENIZERS_PARALLELISM="${TOKENIZERS_PARALLELISM:-false}"export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}"@@ -57,12 +86,12 @@ fi--save_total_limit "${SAVE_TOTAL_LIMIT:-3}" \--dataloader_num_workers "${DATALOADER_NUM_WORKERS:-0}" \--bf16 "${BF16:-true}" \- --gradient_checkpointing "${GRADIENT_CHECKPOINTING:-true}" \+ --gradient_checkpointing "${GRADIENT_CHECKPOINTING:-false}" \--freeze_slow_ar "${FREEZE_SLOW_AR:-false}" \--freeze_fast_ar "${FREEZE_FAST_AR:-false}" \--resume_mode "${RESUME_MODE:-none}" \--report_to "${REPORT_TO:-tensorboard}" \--remove_unused_columns false \- --deepspeed "${DEEPSPEED_CONFIG:-${PROJECT_ROOT}/configs/deepspeed_zero2.json}" \+ --deepspeed "${deepspeed_config}" \"${training_args[@]}" \"$@"diff --git a/configs/deepspeed_zero2.json b/configs/deepspeed_zero2.jsonnew file mode 100644index 0000000..17e90e3--- /dev/null+++ b/configs/deepspeed_zero2.json@@ -0,0 +1,16 @@+{+ "zero_optimization": {+ "stage": 2,+ "allgather_partitions": true,+ "allgather_bucket_size": 5e8,+ "overlap_comm": true,+ "reduce_scatter": true,+ "reduce_bucket_size": 5e8,+ "contiguous_gradients": true+ },+ "gradient_accumulation_steps": "auto",+ "gradient_clipping": "auto",+ "train_batch_size": "auto",+ "train_micro_batch_size_per_gpu": "auto",+ "bf16": {"enabled": "auto"}+}diff --git a/guides/training.md b/guides/training.mdindex e0b4559..d6d785b 100644--- a/guides/training.md+++ b/guides/training.md@@ -2,19 +2,35 @@[All releases](../README.md) · [中文](training_zh.md) · [Inference](inference.md)-These instructions retain the 0.6B SFT workflow. Run from the repository root and explicitly choose the release to fine-tune.+Run from the repository root and explicitly choose the release to fine-tune.```bashexport MODEL=Edge0/Audio8-TTS-Realistic-0.6b# Preview 0.6B: export MODEL=Edge0/Audio8-TTS-Preview-0.6b+# Preview 0.1B: export MODEL=Edge0/Audio8-TTS-Preview-0.1b```+The trainer adapts the slow loss to the selected model's output head:++| Model | Slow loss targets | Gradient checkpointing |+| --- | --- | --- |+| Realistic / Preview 0.6B | Full-vocabulary token IDs, including EOS | Keep disabled; the published model does not support it |+| Preview 0.1B | Semantic classes 0–4095 and EOS class 4096; other tokens ignored | Disabled by default; may be enabled explicitly |++This mapping applies only to the slow loss and accuracy. Model inputs keep their+original token IDs, and fast codebook targets remain unchanged.+Install the training dependencies first:```bashpip install -r requirements-train.txt```+For the 0.1B Falcon-H1 backbone, optional `mamba-ssm` and `causal-conv1d`+packages enable optimized CUDA kernels. Use builds compatible with your+PyTorch/CUDA environment and check startup warnings for kernel fallbacks.+These packages are not needed for 0.6B.+### 1. Create a raw manifestThe target `audio` field is required. `reference_audio` and `reference_text`@@ -66,7 +82,29 @@ For multi-node training, set `NNODES`, `NODE_RANK`, `MASTER_ADDR`, andoverridden through the environment variables in `audio8_tts_sft.sh`; additionalTransformers arguments may be appended to the command.+The launcher includes a [DeepSpeed ZeRO-2 configuration](../configs/deepspeed_zero2.json).+Its `auto` values follow the training arguments. Set `DEEPSPEED_CONFIG=/path/to/config.json`+or append `--deepspeed /path/to/config.json` to use another file; an appended+argument takes precedence. Gradient checkpointing defaults to `false`. For+0.1B only, set `GRADIENT_CHECKPOINTING=true` if needed to reduce activation memory.++By default, each GPU processes 2 samples per microbatch and accumulates gradients+over 8 microbatches: 16 samples per GPU per optimizer step, or 128 samples with+8 GPUs. Step timings should distinguish microsteps from optimizer steps.++### Training objectives and export+SFT optimizes both the slow semantic/EOS objective and the fast codebook-teacher-forcing objective. Set `FREEZE_SLOW_AR=true` or `FREEZE_FAST_AR=true`-when adapting only one branch. The exported directory remains loadable with-standard `AutoModel` and `AutoProcessor` APIs using `trust_remote_code=True`.+teacher-forcing objective. Fast loss covers all 10 codebooks, including the+first semantic codebook as an auxiliary prediction target. It is retained to+match the existing training objective.++Each microbatch contributes a mean slow loss plus a mean fast loss (weighted+by `slow_loss_weight` and `fast_loss_weight`). Gradient accumulation averages+these microbatch losses; it does not reweight them by token count across the+accumulation window. Fast accuracy measures exact codebook-token matches and+is not a direct measure of speech quality.++Set `FREEZE_SLOW_AR=true` or `FREEZE_FAST_AR=true` when adapting only one branch.+The exported directory remains loadable with standard `AutoModel` and+`AutoProcessor` APIs using `trust_remote_code=True`.diff --git a/guides/training_zh.md b/guides/training_zh.mdindex a676553..f3895e5 100644--- a/guides/training_zh.md+++ b/guides/training_zh.md@@ -2,19 +2,33 @@[系列首页](../README_zh.md) · [English](training.md) · [推理](inference_zh.md)-以下保留 0.6B SFT 流程。从仓库根目录执行,并显式选择需要微调的版本。+从仓库根目录执行,并显式选择需要微调的版本。```bashexport MODEL=Edge0/Audio8-TTS-Realistic-0.6b# Preview 0.6B: export MODEL=Edge0/Audio8-TTS-Preview-0.6b+# Preview 0.1B: export MODEL=Edge0/Audio8-TTS-Preview-0.1b```+Trainer 根据所选模型的输出头适配 slow loss:++| 模型 | Slow loss 目标 | Gradient checkpointing |+| --- | --- | --- |+| Realistic / Preview 0.6B | 完整词表中的 token ID,包含 EOS | 保持关闭;已发布模型不支持此功能 |+| Preview 0.1B | Semantic 类别 0–4095,EOS 类别 4096;其余 token 忽略 | 默认关闭,可显式开启 |++该映射仅用于 slow loss 和 accuracy。模型输入保留原始 token ID,fast codebook 目标保持不变。+先安装训练依赖:```bashpip install -r requirements-train.txt```+0.1B 的 Falcon-H1 主干可通过可选的 `mamba-ssm` 和 `causal-conv1d` 包使用优化后的 CUDA+kernel。请选择与 PyTorch/CUDA 环境兼容的构建,并查看启动时是否有 kernel 回退警告。+0.6B 不需要这些包。+### 1. 创建原始数据 manifest目标 `audio` 必填;`reference_audio` 和 `reference_text` 可选,但必须同时出现。@@ -63,6 +77,23 @@ bash audio8_tts_sft.sh和输出路径可通过 `audio8_tts_sft.sh` 中列出的环境变量覆盖,也可以在命令末尾附加 Transformers参数。-SFT 同时优化 slow semantic/EOS loss 和 fast codebook teacher-forcing loss。只训练其中一个分支时,-可设置 `FREEZE_SLOW_AR=true` 或 `FREEZE_FAST_AR=true`。导出目录仍可通过标准 `AutoModel` 和-`AutoProcessor` 接口配合 `trust_remote_code=True` 加载。+启动脚本附带 [DeepSpeed ZeRO-2 配置](../configs/deepspeed_zero2.json),其中的 `auto` 值跟随训练参数。+可设置 `DEEPSPEED_CONFIG=/path/to/config.json`,或在命令末尾附加+`--deepspeed /path/to/config.json` 使用其他配置文件;命令末尾的参数优先。+Gradient checkpointing 默认关闭。仅对 0.1B,可按需设置 `GRADIENT_CHECKPOINTING=true`+以减少 activation 显存占用。++默认每张 GPU 的 microbatch 包含 2 条样本,累积 8 个 microbatch 后更新一次参数,即每个+optimizer step 每张卡处理 16 条样本,8 卡共 128 条。比较耗时前应区分 microstep 和 optimizer step。++### 训练目标与导出++SFT 同时优化 slow semantic/EOS loss 和 fast codebook teacher-forcing loss。Fast loss 保留全部+10 个码本,包括作为辅助预测目标的第一个 semantic 码本,与已有训练目标保持一致。++每个 microbatch 的 loss 为 slow 平均 loss 与 fast 平均 loss 的加权和,权重分别为+`slow_loss_weight` 和 `fast_loss_weight`。梯度累积对这些 microbatch loss 取平均,不按整个累积窗口的+token 总数重新加权。Fast accuracy 衡量码本 token 的精确命中率,并不直接代表语音质量。++只训练其中一个分支时,可设置 `FREEZE_SLOW_AR=true` 或 `FREEZE_FAST_AR=true`。+导出目录仍可通过标准 `AutoModel` 和 `AutoProcessor` 接口配合 `trust_remote_code=True` 加载。diff --git a/tests/test_sft.py b/tests/test_sft.pynew file mode 100644index 0000000..68152e5--- /dev/null+++ b/tests/test_sft.py@@ -0,0 +1,335 @@+"""CPU regression checks for the two SFT output-label contracts.++These tests use small local modules and a deterministic tokenizer, never model+downloads or checkpoint weights. They exercise the public data builder and+Trainer rather than reproducing their implementation.+"""++import re+import sys+from types import SimpleNamespace++import pytest+import torch+import torch.nn.functional as F+from torch import nn+from transformers import TrainingArguments++import audio8_tts_sft as sft+from audio8_tts_data import build_sft_example, collate_sft_examples+++def config_for(family: str) -> SimpleNamespace:+ if family == "0.1b":+ return SimpleNamespace(+ vocab_size=69633,+ codebook_size=4096,+ num_codebooks=10,+ semantic_begin_id=65537,+ semantic_end_id=69632,+ eos_token_id=228,+ )+ return SimpleNamespace(+ vocab_size=155776,+ codebook_size=4096,+ num_codebooks=10,+ semantic_begin_id=151678,+ semantic_end_id=155773,+ eos_token_id=151645,+ )+++def tiny_config() -> SimpleNamespace:+ return SimpleNamespace(+ vocab_size=128,+ codebook_size=4,+ num_codebooks=10,+ semantic_begin_id=100,+ semantic_end_id=103,+ eos_token_id=2,+ pad_token_id=0,+ norm_fastlayer_input=False,+ )+++def test_compact_targets_map_semantic_and_eos_without_mutating_input():+ config = config_for("0.1b")+ labels = torch.tensor([[-100, 65537, 69632, 228, 709, 17]])+ original = labels.clone()++ targets = sft.slow_loss_targets(labels, config, 4097)++ assert targets.tolist() == [[-100, 0, 4095, 4096, -100, -100]]+ assert torch.equal(labels, original)+ assert torch.isfinite(F.cross_entropy(torch.zeros((6, 4097)), targets.reshape(-1)))+++def test_full_vocabulary_targets_preserve_existing_loss_and_gradients():+ config = config_for("0.6b")+ labels = torch.tensor([[-100, 151678, 155773, 151645, 198]])+ generator = torch.Generator().manual_seed(7)+ logits_before = torch.randn((5, config.vocab_size), generator=generator).requires_grad_()+ logits_after = logits_before.detach().clone().requires_grad_()++ targets = sft.slow_loss_targets(labels, config, config.vocab_size)+ loss_before = F.cross_entropy(logits_before, labels.reshape(-1))+ loss_after = F.cross_entropy(logits_after, targets.reshape(-1))+ loss_before.backward()+ loss_after.backward()++ assert torch.equal(targets, labels)+ torch.testing.assert_close(loss_after, loss_before, rtol=0, atol=0)+ torch.testing.assert_close(logits_after.grad, logits_before.grad, rtol=0, atol=0)+++def test_unknown_slow_output_contract_is_rejected():+ config = config_for("0.1b")+ with pytest.raises(ValueError):+ sft.slow_loss_targets(torch.tensor([[65537, 228]]), config, 4098)+++class PromptTokenizer:+ """Small tokenizer that keeps chat delimiters distinct for builder tests."""++ special = {"<|im_start|>": 1, "<|im_end|>": 2, "<|voice|>": 3, "<|speaker:0|>": 4}++ def encode(self, text, add_special_tokens=False):+ assert add_special_tokens is False+ pieces = re.findall(r"<\|[^|]+\|>|.", text, flags=re.DOTALL)+ return [self.special.get(piece, 7 if piece == "\n" else 5) for piece in pieces]+++def codec_frames(semantic):+ semantic = torch.tensor(semantic, dtype=torch.long)+ return torch.stack([(semantic + index) % 4 for index in range(10)])+++def test_builder_preserves_reference_masking_shift_eos_and_padding():+ config = tiny_config()+ target = codec_frames([0, 1, 3])+ example = build_sft_example(+ tokenizer=PromptTokenizer(),+ config=config,+ text="Hello.",+ target_codes=target,+ reference_codes=codec_frames([2, 2]),+ reference_text="Reference.",+ )+ labels = example["labels"]+ semantic_positions = labels[0].ge(100) & labels[0].le(103)++ # Three target frames are supervised; the two reference frames are inputs+ # only. The next input column is the target predicted at this position.+ assert labels[0, semantic_positions].tolist() == [100, 101, 103]+ assert torch.equal(labels[1:, semantic_positions], target)+ input_semantic = example["input_ids"][0].ge(100) & example["input_ids"][0].le(103)+ assert int(input_semantic.sum()) == 5+ for position in torch.where(semantic_positions)[0].tolist():+ assert torch.equal(example["input_ids"][:, position + 1], labels[:, position])+ assert labels[0, labels[0].ne(-100)].tolist() == [100, 101, 103, 2, 7]+ assert labels[1:, ~semantic_positions].eq(-100).all()++ shorter = build_sft_example(+ tokenizer=PromptTokenizer(), config=config, text="Hi.", target_codes=codec_frames([1])+ )+ batch = collate_sft_examples([example, shorter], pad_token_id=config.pad_token_id)+ padding = batch["attention_mask"].eq(0)+ assert padding.any()+ assert batch["labels"].permute(0, 2, 1)[padding].eq(-100).all()+ assert batch["input_ids"][:, 0][padding].eq(config.pad_token_id).all()+ compact = sft.slow_loss_targets(batch["labels"][:, 0], config, 5)+ assert compact[0, compact[0].ne(-100)].tolist() == [0, 1, 3, 4]+ assert compact[padding].eq(-100).all()+++def training_args(path, accumulation=1):+ return TrainingArguments(+ output_dir=str(path),+ use_cpu=True,+ report_to=[],+ save_strategy="no",+ logging_strategy="no",+ disable_tqdm=True,+ dataloader_pin_memory=False,+ gradient_accumulation_steps=accumulation,+ )+++class TinyDualAR(nn.Module):+ """Minimal model exposing the actual slow/fast Trainer interface."""++ def __init__(self, compact):+ super().__init__()+ self.config = tiny_config()+ classes = 5 if compact else 128+ logits = torch.zeros((1, 6, classes))+ expected = [0, 0, 3, 4, 1, 0] if compact else [0, 100, 103, 2, 7, 0]+ for position, prediction in enumerate(expected):+ logits[0, position, prediction] = 6+ self.slow_logits = nn.Parameter(logits)+ self.slow_hidden = nn.Parameter(torch.ones((1, 6, 4)))+ self.fast_project_in = nn.Identity()+ self.fast_embeddings = nn.Embedding(4, 4)+ self.fast_layers = nn.ModuleList()+ self.fast_norm = nn.Identity()+ self.fast_output = nn.Linear(4, 4, bias=False)+ self.register_buffer("fast_freqs_cis", torch.zeros((10, 4)))++ @staticmethod+ def _causal_mask(attention_mask, positions, length):+ return torch.ones((length, length), dtype=torch.bool).tril()[None, None]++ def forward(self, input_ids, attention_mask=None, return_dict=True, **kwargs):+ return SimpleNamespace(logits=self.slow_logits, hidden_states=self.slow_hidden)++[email protected]("compact", [False, True], ids=["full-vocabulary", "compact"])+def test_trainer_uses_correct_slow_accuracy_and_keeps_all_fast_targets(+ tmp_path, monkeypatch, compact+):+ model = TinyDualAR(compact)+ trainer = sft.Audio8TTSTrainer(+ model=model,+ args=training_args(tmp_path),+ slow_loss_weight=1.0,+ fast_loss_weight=1.0,+ )+ labels = torch.full((1, 11, 6), -100, dtype=torch.long)+ labels[0, 0] = torch.tensor([-100, 100, 103, 2, 7, -100])+ expected_codebooks = codec_frames([0, 3]).T+ labels[0, 1:, 1:3] = expected_codebooks.T+ original = labels.clone()+ seen = []+ original_fast_logits = sft.fast_codebook_logits++ def capture_fast_inputs(core, hidden, codebooks):+ seen.append(codebooks.detach().clone())+ return original_fast_logits(core, hidden, codebooks)++ monkeypatch.setattr(sft, "fast_codebook_logits", capture_fast_inputs)+ loss, outputs = trainer.compute_loss(+ model,+ {"input_ids": torch.zeros((1, 11, 6), dtype=torch.long), "labels": labels},+ return_outputs=True,+ )+ assert torch.isfinite(loss)+ assert torch.equal(labels, original)+ assert len(seen) == 1+ assert torch.equal(seen[0], expected_codebooks)+ assert outputs["codebook_logits"].shape == (2, 10, 4)+ assert trainer._metric_sums["slow_accuracy"] == pytest.approx(1.0)+ assert trainer._metric_sums["semantic_frames"] == 2+ expected_fast_loss = F.cross_entropy(+ outputs["codebook_logits"].reshape(-1, 4), expected_codebooks.reshape(-1)+ )+ assert trainer._metric_sums["fast_loss"] == pytest.approx(float(expected_fast_loss.detach()))+ loss.backward()+ assert model.slow_logits.grad is not None+ assert model.slow_hidden.grad is not None+ assert model.slow_hidden.grad[:, 1:3].abs().sum() > 0+++def test_main_rejects_unsupported_checkpointing_before_dataset_access(tmp_path, monkeypatch):+ model = TinyDualAR(compact=False)+ model.supports_gradient_checkpointing = False+ model.config.max_seq_len = 2048+ monkeypatch.setattr(+ sft.AutoProcessor,+ "from_pretrained",+ lambda *args, **kwargs: SimpleNamespace(tokenizer=PromptTokenizer()),+ )+ monkeypatch.setattr(sft.AutoModel, "from_pretrained", lambda *args, **kwargs: model)+ monkeypatch.setattr(+ sys,+ "argv",+ [+ "audio8_tts_sft.py",+ "--model_name_or_path",+ "local-test-model",+ "--train_jsonl",+ str(tmp_path / "does-not-exist.jsonl"),+ "--output_dir",+ str(tmp_path / "output"),+ "--gradient_checkpointing",+ "true",+ "--use_cpu",+ "true",+ "--report_to",+ "none",+ ],+ )+ with pytest.raises(ValueError, match="does not support gradient checkpointing"):+ sft.main()+++class Branches(nn.Module):+ def __init__(self, family):+ super().__init__()+ if family == "0.1b":+ self.slow = nn.Linear(2, 2)+ self.semantic_output = nn.Linear(2, 2)+ else:+ self.embeddings = nn.Embedding(4, 2)+ self.layers = nn.ModuleList([nn.Linear(2, 2)])+ self.norm = nn.LayerNorm(2)+ self.codebook_embeddings = nn.Embedding(4, 2)+ self.fast_project_in = nn.Linear(2, 2)+ self.fast_embeddings = nn.Embedding(4, 2)+ self.fast_layers = nn.ModuleList([nn.Linear(2, 2)])+ self.fast_norm = nn.LayerNorm(2)+ self.fast_output = nn.Linear(2, 2)++[email protected]("family", ["0.1b", "0.6b"])[email protected]("freeze_slow,freeze_fast", [(False, False), (True, False), (False, True)])+def test_freeze_flags_cover_both_architectures(family, freeze_slow, freeze_fast):+ model = Branches(family)+ fast_parameters = {+ id(parameter) for name, parameter in model.named_parameters() if name.startswith("fast_")+ }+ sft.set_trainable_modules(model, freeze_slow=freeze_slow, freeze_fast=freeze_fast)+ for parameter in model.parameters():+ should_freeze = freeze_fast if id(parameter) in fast_parameters else freeze_slow+ assert parameter.requires_grad is not should_freeze++[email protected]("family", ["0.1b", "0.6b"])+def test_freezing_every_branch_is_rejected(family):+ with pytest.raises(ValueError, match="no trainable parameters"):+ sft.set_trainable_modules(Branches(family), freeze_slow=True, freeze_fast=True)+++class ScalarMeanModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.weight = nn.Parameter(torch.tensor(1.0))++ def forward(self, values, **kwargs):+ # **kwargs makes Transformers infer loss-kwargs support unless the+ # Audio8 Trainer explicitly declares its microbatch-mean interface.+ return self.weight * values+++class ScalarMeanTrainer(sft.Audio8TTSTrainer):+ def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):+ return model(inputs["values"]).square().mean()+++def test_trainer_mean_loss_accumulation_matches_equal_sized_batch(tmp_path):+ gradients = []+ values = torch.tensor([1.0, 2.0, 3.0, 4.0])+ for accumulation in (1, 4):+ model = ScalarMeanModel()+ trainer = ScalarMeanTrainer(+ model=model,+ args=training_args(tmp_path / str(accumulation), accumulation),+ slow_loss_weight=1.0,+ fast_loss_weight=1.0,+ )+ trainer.current_gradient_accumulation_steps = accumulation+ for microbatch in values.chunk(accumulation):+ trainer.training_step(model, {"values": microbatch}, num_items_in_batch=torch.tensor(4))+ gradients.append(model.weight.grad.detach().clone())++ torch.testing.assert_close(gradients[0], torch.tensor(15.0))+ torch.testing.assert_close(gradients[1], gradients[0])diff --git a/tests/test_sft_launcher.py b/tests/test_sft_launcher.pynew file mode 100644index 0000000..01acaa7--- /dev/null+++ b/tests/test_sft_launcher.py@@ -0,0 +1,101 @@+"""Validate launcher arguments without importing torch or starting training."""++import argparse+import json+import os+import shlex+import subprocess+import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]+++def launch(tmp_path):+ capture = tmp_path / "capture.py"+ capture.write_text("import json, sys\nprint(json.dumps(sys.argv[1:]))\n")+ fake_python = tmp_path / "fake python"+ fake_python.write_text(+ f'#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(capture))} "$@"\n'+ )+ fake_python.chmod(0o755)++ def run(args=(), **overrides):+ env = {+ "PATH": os.environ["PATH"],+ "PYTHON": str(fake_python),+ "TRAIN_JSONL": "prepared_data/train.jsonl",+ **overrides,+ }+ return subprocess.run(+ ["bash", str(ROOT / "audio8_tts_sft.sh"), *args],+ cwd=tmp_path,+ env=env,+ text=True,+ capture_output=True,+ timeout=10,+ )++ return run+++def captured_options(result):+ assert result.returncode == 0, result.stderr+ argv = json.loads(result.stdout)+ assert argv[:2] == ["-m", "torch.distributed.run"]+ parser = argparse.ArgumentParser(allow_abbrev=False)+ parser.add_argument("--deepspeed")+ parser.add_argument("--gradient_checkpointing")+ options, _ = parser.parse_known_args(argv)+ return options+++def test_defaults_use_bundled_zero2_and_disable_checkpointing(launch):+ options = captured_options(launch())+ config = ROOT / "configs" / "deepspeed_zero2.json"+ assert options.deepspeed == str(config)+ assert json.loads(config.read_text())["zero_optimization"]["stage"] == 2+ assert options.gradient_checkpointing == "false"++[email protected]("source", ["env", "separate", "equals"])+def test_custom_config_overrides_default(launch, tmp_path, source):+ config = tmp_path / "custom config.json"+ config.write_text('{"zero_optimization": {"stage": 2}}')+ env = {"DEEPSPEED_CONFIG": str(tmp_path / "missing.json")}+ args = []+ if source == "env":+ env["DEEPSPEED_CONFIG"] = str(config)+ elif source == "separate":+ args = ["--deepspeed", str(config)]+ else:+ args = [f"--deepspeed={config}"]+ assert captured_options(launch(args, **env)).deepspeed == str(config)+++ "args",+ [+ [],+ ["--deepspeed", "missing.json"],+ ["--deepspeed"],+ ["--deepspeed="],+ ["--deepspeed", "--bf16", "false"],+ ],+)+def test_bad_config_fails_before_invoking_python(launch, tmp_path, args):+ result = launch(args, DEEPSPEED_CONFIG=str(tmp_path / "missing.json"))+ assert result.returncode == 2+ assert result.stdout == ""+ assert (+ "DeepSpeed config is not a readable file:" in result.stderr+ or "--deepspeed requires a JSON config file path." in result.stderr+ )+++def test_checkpointing_can_be_explicitly_enabled(launch):+ options = captured_options(launch(GRADIENT_CHECKPOINTING="true"))+ assert options.gradient_checkpointing == "true"