SOTA Open Source TTS
import re
import unittest
from unittest.mock import patch
import torch
from torch.nn import functional as F
from transformers.models.qwen3_5_moe.configuration_qwen3_5_moe import (
Qwen3_5MoeTextConfig,
)
from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeForCausalLM
from fish_speech.models.text2semantic.fish_omni import fp8_moe
from fish_speech.models.text2semantic.fish_omni.backbone import (
FishOmniBackbone,
load_backbone_weights,
)
from fish_speech.models.text2semantic.fish_omni.config import FishOmniTextConfig
from fish_speech.models.text2semantic.fish_omni.fp8_moe import (
FP8_BLOCK_SIZE,
FP8_DTYPE,
FP8_MAX,
fp8_gemm_supported,
fp8_moe_forward,
)
from fish_speech.models.text2semantic.fish_omni.layers import (
chunk_gated_delta_rule,
recurrent_gated_delta_rule,
)
from fish_speech.models.text2semantic.fish_omni.moe import (
MoEExperts,
dequantize_fp8_blocks,
)
VOCAB_SIZE = 96
SEQUENCE_LENGTH = 150 # crosses two GDN chunk boundaries
def tiny_hf_config() -> Qwen3_5MoeTextConfig:
config = Qwen3_5MoeTextConfig(
vocab_size=VOCAB_SIZE,
hidden_size=64,
num_hidden_layers=4,
layer_types=[
"linear_attention",
"full_attention",
"linear_attention",
"full_attention",
],
num_attention_heads=4,
num_key_value_heads=2,
head_dim=16,
linear_conv_kernel_dim=4,
linear_key_head_dim=8,
linear_value_head_dim=8,
linear_num_key_heads=2,
linear_num_value_heads=4,
num_experts=8,
num_experts_per_tok=2,
moe_intermediate_size=32,
shared_expert_intermediate_size=32,
max_position_embeddings=512,
rms_norm_eps=1e-6,
rope_parameters={
"rope_type": "default",
"rope_theta": 10000.0,
"partial_rotary_factor": 0.5,
},
)
config._attn_implementation = "eager"
return config
def randomize_fp8_experts(experts: MoEExperts) -> tuple[torch.Tensor, torch.Tensor]:
"""Fill FP8 experts with random weights; returns the dequantized w13 and w2."""
device = experts.w13_weight.device
for name in ("w13", "w2"):
weight = getattr(experts, f"{name}_weight")
scale = getattr(experts, f"{name}_scale_inv")
weight.copy_(torch.randn(weight.shape, device=device).to(FP8_DTYPE))
scale.copy_(torch.rand(scale.shape, device=device) * 0.01)
return (
dequantize_fp8_blocks(experts.w13_weight, experts.w13_scale_inv, torch.float32),
dequantize_fp8_blocks(experts.w2_weight, experts.w2_scale_inv, torch.float32),
)
def fp8_checkpoint(
state_dict: dict[str, torch.Tensor],
) -> list[tuple[str, torch.Tensor]]:
"""state_dict with fused HF experts rewritten as per-expert block-FP8 tensors."""
checkpoint = []
for name, tensor in state_dict.items():
match = re.fullmatch(
r"model\.layers\.(\d+)\.mlp\.experts\.(gate_up_proj|down_proj)", name
)
if match is None:
checkpoint.append((name, tensor))
continue
for expert, weight in enumerate(tensor):
if match[2] == "gate_up_proj":
projections = list(zip(("gate_proj", "up_proj"), weight.chunk(2)))
else:
projections = [("down_proj", weight)]
for projection, matrix in projections:
rows, cols = matrix.shape
blocks = matrix.reshape(
rows // FP8_BLOCK_SIZE,
FP8_BLOCK_SIZE,
cols // FP8_BLOCK_SIZE,
FP8_BLOCK_SIZE,
)
scale = blocks.abs().amax(dim=(1, 3)) / FP8_MAX
quantized = (blocks / scale[:, None, :, None]).reshape(rows, cols)
key = f"model.layers.{match[1]}.mlp.experts.{expert}.{projection}"
checkpoint.append((f"{key}.weight", quantized.to(FP8_DTYPE)))
checkpoint.append((f"{key}.weight_scale_inv", scale.float()))
return checkpoint
class BackboneMatchesHuggingFaceTest(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
torch.manual_seed(0)
hf_config = tiny_hf_config()
cls.reference = Qwen3_5MoeForCausalLM(hf_config).eval()
with torch.no_grad():
for parameter in cls.reference.parameters():
parameter.normal_(0.0, 0.1 if parameter.ndim >= 2 else 0.3)
cls.config = FishOmniTextConfig.from_dict(hf_config.to_dict())
cls.tokens = torch.randint(0, VOCAB_SIZE, (2, SEQUENCE_LENGTH))
with torch.no_grad():
cls.expected = cls.reference(cls.tokens, use_cache=False).logits
def backbone(self) -> FishOmniBackbone:
model = FishOmniBackbone(self.config, dtype=torch.float32, device="cpu").eval()
load_backbone_weights(model, self.reference.state_dict().items())
return model
def assert_logits_close(self, actual: torch.Tensor, expected: torch.Tensor) -> None:
torch.testing.assert_close(actual, expected, rtol=1e-4, atol=2e-4)
@torch.no_grad()
def test_whole_sequence_without_cache(self) -> None:
model = self.backbone()
logits = model.compute_logits(model(self.tokens))
self.assert_logits_close(logits, self.expected)
@torch.no_grad()
def test_cached_prefill_decode_and_chunked_continuation(self) -> None:
model = self.backbone()
model.setup_caches(max_batch_size=2, max_seq_len=256)
# Prefill, single-token decodes, a multi-token continuation, more decodes.
segments = [(0, 70), *[(p, p + 1) for p in range(70, 75)], (75, 95)]
segments += [(p, p + 1) for p in range(95, 98)]
for start, end in segments:
hidden = model(
self.tokens[:, start:end], input_pos=torch.arange(start, end)
)
self.assert_logits_close(
model.compute_logits(hidden), self.expected[:, start:end]
)
@torch.no_grad()
def test_new_sequence_overwrites_cached_state(self) -> None:
model = self.backbone()
model.setup_caches(max_batch_size=2, max_seq_len=256)
model(self.tokens[:, :40].flip(0), input_pos=torch.arange(40))
hidden = model(self.tokens[:, :30], input_pos=torch.arange(30))
next_hidden = model(self.tokens[:, 30:31], input_pos=torch.tensor([30]))
hidden = torch.cat([hidden, next_hidden], dim=1)
self.assert_logits_close(model.compute_logits(hidden), self.expected[:, :31])
@torch.no_grad()
def test_rejects_gaps_and_continuation_without_cache(self) -> None:
model = self.backbone()
with self.assertRaisesRegex(AssertionError, "setup_caches"):
model(self.tokens[:, :2], input_pos=torch.tensor([3, 4]))
model.setup_caches(max_batch_size=2, max_seq_len=16)
with self.assertRaisesRegex(AssertionError, "contiguous"):
model(self.tokens[:, :2], input_pos=torch.tensor([0, 2]))
with self.assertRaisesRegex(AssertionError, "exceed the caches"):
model(self.tokens[:, :17], input_pos=torch.arange(17))
def test_loader_requires_every_slot_once(self) -> None:
state = list(self.reference.state_dict().items())
with self.assertRaisesRegex(ValueError, "not loaded"):
load_backbone_weights(
FishOmniBackbone(self.config, dtype=torch.float32, device="cpu"),
[item for item in state if item[0] != "model.norm.weight"],
)
with self.assertRaisesRegex(ValueError, "filled twice"):
load_backbone_weights(
FishOmniBackbone(self.config, dtype=torch.float32, device="cpu"),
state + state[:1],
)
class RotaryConfigTest(unittest.TestCase):
def test_partial_rotary_factor_matches_hf(self) -> None:
data = tiny_hf_config().to_dict()
rope = data["rope_parameters"]
self.assertEqual(FishOmniTextConfig.from_dict(data).partial_rotary_factor, 0.5)
factor = rope.pop("partial_rotary_factor")
data.pop("partial_rotary_factor", None)
self.assertEqual(FishOmniTextConfig.from_dict(data).partial_rotary_factor, 0.25)
data["partial_rotary_factor"] = factor
self.assertEqual(
FishOmniTextConfig.from_dict(data).partial_rotary_factor, factor
)
class GatedDeltaRuleTest(unittest.TestCase):
@torch.no_grad()
def test_chunked_matches_recurrent_with_initial_state(self) -> None:
torch.manual_seed(1)
batch, length, heads, key_dim, value_dim = 2, 133, 3, 8, 6
query = torch.randn(batch, length, heads, key_dim)
key = torch.randn(batch, length, heads, key_dim)
value = torch.randn(batch, length, heads, value_dim)
decay = -torch.rand(batch, length, heads)
beta = torch.rand(batch, length, heads)
state = torch.randn(batch, heads, key_dim, value_dim)
chunked = chunk_gated_delta_rule(query, key, value, decay, beta, state)
recurrent = recurrent_gated_delta_rule(query, key, value, decay, beta, state)
torch.testing.assert_close(chunked[0], recurrent[0], rtol=1e-4, atol=1e-4)
torch.testing.assert_close(chunked[1], recurrent[1], rtol=1e-4, atol=1e-4)
class Fp8ExpertsTest(unittest.TestCase):
@torch.no_grad()
def setUp(self) -> None:
torch.manual_seed(2)
hidden, intermediate, count = 2 * FP8_BLOCK_SIZE, FP8_BLOCK_SIZE, 6
self.experts = MoEExperts(count, hidden, intermediate, fp8=True, device="cpu")
self.w13, self.w2 = randomize_fp8_experts(self.experts)
def test_gemm_gate_needs_sm89_cuda_not_rocm(self) -> None:
with (
patch.object(fp8_moe, "triton", object()),
patch("torch.cuda.is_available", return_value=True),
patch("torch.cuda.get_device_capability", return_value=(9, 4)),
):
with patch.object(torch.version, "hip", None):
self.assertTrue(fp8_gemm_supported(["cuda:0"]))
with patch.object(torch.version, "hip", "7.2.0"):
self.assertFalse(fp8_gemm_supported(["cuda:0"]))
with (
patch.object(fp8_moe, "triton", object()),
patch("torch.cuda.is_available", return_value=True),
patch("torch.cuda.get_device_capability", return_value=(8, 0)),
patch.object(torch.version, "hip", None),
):
self.assertFalse(fp8_gemm_supported(["cuda:0"]))
def test_block_dequantization(self) -> None:
experts = self.experts
row, col = FP8_BLOCK_SIZE + 3, 2 * FP8_BLOCK_SIZE - 1
expected = (
experts.w13_weight[4, row, col].float()
* experts.w13_scale_inv[4, row // FP8_BLOCK_SIZE, col // FP8_BLOCK_SIZE]
)
torch.testing.assert_close(self.w13[4, row, col], expected)
@torch.no_grad()
def test_gathered_and_grouped_paths_match_reference(self) -> None:
x = torch.randn(5, self.experts.hidden)
topk_ids = torch.tensor([[0, 3], [3, 5], [1, 0], [5, 5], [2, 4]])
topk_weights = torch.rand(5, 2)
expected = torch.zeros_like(x)
for token in range(5):
for slot in range(2):
expert = topk_ids[token, slot]
gate, up = (self.w13[expert] @ x[token]).chunk(2)
expected[token] += topk_weights[token, slot] * (
self.w2[expert] @ (F.silu(gate) * up)
)
for path in (self.experts.forward_gathered, self.experts.forward_grouped):
torch.testing.assert_close(
path(x, topk_ids, topk_weights), expected, rtol=1e-4, atol=1e-4
)
@unittest.skipUnless(fp8_gemm_supported(["cuda:0"]), "needs an SM89+ GPU")
class NativeFp8ExpertsTest(unittest.TestCase):
"""The W8A8 kernel against a reference with the same activation quantization."""
@staticmethod
def fake_quant(v: torch.Tensor) -> torch.Tensor:
groups = v.float().reshape(v.shape[0], -1, FP8_BLOCK_SIZE)
scale = groups.abs().amax(-1, keepdim=True).clamp(min=1e-10) / FP8_MAX
return ((groups / scale).to(FP8_DTYPE).float() * scale).reshape(v.shape)
@torch.no_grad()
def test_matches_quantized_reference(self) -> None:
torch.manual_seed(4)
device = "cuda:0"
count, top_k = 16, 4
hidden, intermediate = 2 * FP8_BLOCK_SIZE, 3 * FP8_BLOCK_SIZE
experts = MoEExperts(count, hidden, intermediate, fp8=True, device=device)
w13, w2 = randomize_fp8_experts(experts)
for tokens in (1, 7, 300):
x = torch.randn(tokens, hidden, device=device, dtype=torch.bfloat16)
weights, ids = torch.rand(tokens, count, device=device).topk(top_k, dim=-1)
weights = weights.to(torch.bfloat16)
x_q = self.fake_quant(x)
expected = torch.zeros(tokens, hidden, device=device)
for slot in range(top_k):
e = ids[:, slot]
gate_up = torch.einsum("tnk,tk->tn", w13[e], x_q).bfloat16().float()
gate, up = gate_up.chunk(2, dim=-1)
h = self.fake_quant(F.silu(gate) * up)
expected += weights[:, slot, None].float() * torch.einsum(
"tnk,tk->tn", w2[e], h
)
got = experts(x, ids, weights).float()
self.assertEqual(got.shape, (tokens, hidden))
relative = (got - expected).norm() / expected.norm()
self.assertLess(relative.item(), 1e-2, f"tokens={tokens}")
@unittest.skipUnless(torch.cuda.device_count() > 1, "needs two GPUs")
@torch.no_grad()
def test_runs_on_a_non_current_device(self) -> None:
device = "cuda:1"
count, hidden, intermediate = 4, FP8_BLOCK_SIZE, FP8_BLOCK_SIZE
w13 = torch.randn(count, 2 * intermediate, hidden, device=device)
w2 = torch.randn(count, hidden, intermediate, device=device)
s13 = torch.full((count, 2, 1), 0.01, device=device)
s2 = torch.full((count, 1, 1), 0.01, device=device)
x = torch.randn(3, hidden, device=device, dtype=torch.bfloat16)
ids = torch.tensor([[0, 1], [2, 3], [1, 1]], device=device)
weights = torch.full((3, 2), 0.5, device=device, dtype=torch.bfloat16)
with torch.cuda.device(0):
out = fp8_moe_forward(
x, ids, weights, w13.to(FP8_DTYPE), s13, w2.to(FP8_DTYPE), s2
)
self.assertEqual(out.device, torch.device(device))
self.assertTrue(torch.isfinite(out).all())
class Fp8CheckpointLoadTest(unittest.TestCase):
@torch.no_grad()
def test_dense_and_fp8_experts_agree(self) -> None:
"""Per-expert FP8 tensors load into dense (dequantized) and FP8 experts alike."""
torch.manual_seed(3)
hf_config = tiny_hf_config()
hf_config.hidden_size = FP8_BLOCK_SIZE
hf_config.moe_intermediate_size = FP8_BLOCK_SIZE
reference = Qwen3_5MoeForCausalLM(hf_config).eval()
for parameter in reference.parameters():
parameter.normal_(0.0, 0.1 if parameter.ndim >= 2 else 0.3)
config = FishOmniTextConfig.from_dict(hf_config.to_dict())
checkpoint = fp8_checkpoint(reference.state_dict())
tokens = torch.randint(0, VOCAB_SIZE, (1, 40))
logits = {}
for keep_fp8 in (False, True):
model = FishOmniBackbone(
config, fp8_experts=keep_fp8, dtype=torch.float32, device="cpu"
).eval()
load_backbone_weights(model, checkpoint)
logits[keep_fp8] = model.compute_logits(model(tokens))
torch.testing.assert_close(logits[False], logits[True], rtol=1e-5, atol=1e-5)
# The reference with its experts replaced by the same dequantized values.
dequantized = dict(checkpoint)
for layer_index, layer in enumerate(reference.model.layers):
experts = layer.mlp.experts
for expert in range(experts.gate_up_proj.shape[0]):
key = f"model.layers.{layer_index}.mlp.experts.{expert}"
weights = {
projection: dequantize_fp8_blocks(
dequantized[f"{key}.{projection}.weight"],
dequantized[f"{key}.{projection}.weight_scale_inv"],
torch.float32,
)
for projection in ("gate_proj", "up_proj", "down_proj")
}
experts.gate_up_proj[expert] = torch.cat(
(weights["gate_proj"], weights["up_proj"])
)
experts.down_proj[expert] = weights["down_proj"]
expected = reference(tokens, use_cache=False).logits
torch.testing.assert_close(logits[False], expected, rtol=1e-4, atol=2e-4)
missing_scale = "model.layers.0.mlp.experts.0.down_proj.weight_scale_inv"
incomplete = [item for item in checkpoint if item[0] != missing_scale]
with self.assertRaisesRegex(ValueError, "lack a weight or scale"):
load_backbone_weights(
FishOmniBackbone(config, dtype=torch.float32, device="cpu"), incomplete
)
def test_float16_keeps_a_float32_residual_stream(self) -> None:
config = FishOmniTextConfig.from_dict(tiny_hf_config().to_dict())
with torch.device("meta"):
half = FishOmniBackbone(config, dtype=torch.float16)
bfloat = FishOmniBackbone(config, dtype=torch.bfloat16)
self.assertEqual(half.residual_dtype, torch.float32)
self.assertEqual(bfloat.residual_dtype, torch.bfloat16)
if __name__ == "__main__":
unittest.main()