Skip to content

SOTA Open Source TTS

16.4 KB
Raw Blame
import copy
import unittest
from unittest.mock import patch

import torch
from torch import nn
from torch.nn import functional as F

from fish_speech.models.text2semantic.inference import (
    decode_one_token_ar,
    generate,
    generated_codes,
)
from fish_speech.models.text2semantic.llama import (
    Attention,
    BaseModelArgs,
    DualARModelArgs,
    DualARTransformer,
    KVCache,
    NaiveModelArgs,
    NaiveTransformer,
    precompute_freqs_cis,
)

CACHE_CAPACITY = 8
PROMPT_LENGTH = 3


class _TokenizerStub:
    eos_token_id = 15

    def get_token_id(self, _token: str) -> int:
        return self.eos_token_id


class _ConstantOutput(nn.Module):
    def __init__(self, size: int, preferred_id: int, suppressed_id: int | None = None):
        super().__init__()
        logits = torch.zeros(size)
        logits[preferred_id] = 10.0
        if suppressed_id is not None:
            logits[suppressed_id] = -10.0
        self.register_buffer("logits", logits)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.logits.to(dtype=x.dtype).expand(*x.shape[:-1], -1)


def _base_args(**overrides) -> dict:
    args = {
        "vocab_size": 16,
        "n_layer": 1,
        "n_head": 2,
        "n_local_heads": 1,
        "dim": 8,
        "intermediate_size": 16,
        "head_dim": 4,
        "max_seq_len": CACHE_CAPACITY,
        "dropout": 0.0,
        "tie_word_embeddings": False,
        "codebook_size": 8,
        "num_codebooks": 2,
        "semantic_begin_id": 4,
        "semantic_end_id": 11,
    }
    args.update(overrides)
    return args


class ActiveKVAttentionTest(unittest.TestCase):
    @staticmethod
    def _attention() -> Attention:
        config = BaseModelArgs(**_base_args())
        attention = Attention(config).eval()
        attention.kv_cache = KVCache(
            max_batch_size=1,
            max_seq_len=CACHE_CAPACITY,
            n_heads=1,
            head_dim=4,
            dtype=torch.float32,
        )
        return attention

    @staticmethod
    def _seed_cache(attention: Attention) -> None:
        torch.manual_seed(100)
        attention.kv_cache.k_cache[:, :, :2] = torch.randn(1, 1, 2, 4)
        attention.kv_cache.v_cache[:, :, :2] = torch.randn(1, 1, 2, 4)

    def test_attention_passes_only_active_kv_prefix_to_sdpa(self) -> None:
        torch.manual_seed(1)
        attention = self._attention()
        self._seed_cache(attention)
        x = torch.randn(1, 1, 8)
        freqs_cis = precompute_freqs_cis(CACHE_CAPACITY, 4)[2:3]
        mask = torch.ones(1, 1, 1, 3, dtype=torch.bool)
        original_sdpa = F.scaled_dot_product_attention
        observed_shapes = []

        def capture_sdpa(query, key, value, **kwargs):
            observed_shapes.append((key.shape, value.shape))
            return original_sdpa(query, key, value, **kwargs)

        with patch(
            "fish_speech.models.text2semantic.llama.F.scaled_dot_product_attention",
            side_effect=capture_sdpa,
        ):
            attention(x, freqs_cis, mask, input_pos=torch.tensor([2]))

        self.assertEqual(
            observed_shapes,
            [(torch.Size([1, 2, 3, 4]), torch.Size([1, 2, 3, 4]))],
        )
        self.assertEqual(attention.kv_cache.k_cache.shape[2], CACHE_CAPACITY)

    def test_active_prefix_matches_masked_full_cache_with_tolerance(self) -> None:
        torch.manual_seed(2)
        active_attention = self._attention()
        self._seed_cache(active_attention)
        full_attention = copy.deepcopy(active_attention)
        x = torch.randn(1, 1, 8)
        freqs_cis = precompute_freqs_cis(CACHE_CAPACITY, 4)[2:3]
        active_mask = torch.ones(1, 1, 1, 3, dtype=torch.bool)
        full_mask = torch.zeros(1, 1, 1, CACHE_CAPACITY, dtype=torch.bool)
        full_mask[..., :3] = True
        input_pos = torch.tensor([2])

        active_output = active_attention(
            x,
            freqs_cis,
            active_mask,
            input_pos=input_pos,
        )
        full_output = full_attention(
            x,
            freqs_cis,
            full_mask,
            input_pos=input_pos,
        )

        torch.testing.assert_close(active_output, full_output, rtol=1e-5, atol=1e-6)
        torch.testing.assert_close(
            active_output.argmax(dim=-1),
            full_output.argmax(dim=-1),
            rtol=0,
            atol=0,
        )


class ActiveKVProgressionTest(unittest.TestCase):
    @staticmethod
    def _model() -> NaiveTransformer:
        torch.manual_seed(3)
        model = NaiveTransformer(NaiveModelArgs(**_base_args())).eval()
        model.setup_caches(1, CACHE_CAPACITY, dtype=torch.float32)
        return model

    @staticmethod
    def _tokens(length: int) -> torch.Tensor:
        main = torch.arange(1, length + 1).remainder(4).view(1, 1, length)
        codebooks = torch.arange(2 * length).remainder(8).view(1, 2, length)
        return torch.cat((main, codebooks), dim=1)

    def test_kv_len_validation(self) -> None:
        model = self._model()
        token = self._tokens(1)
        input_pos = torch.tensor([0])

        minimum_prefix = model.forward_generate(
            token,
            input_pos=input_pos,
            kv_len=1,
        )
        self.assertTrue(torch.isfinite(minimum_prefix.token_logits).all())
        self.assertTrue(torch.isfinite(minimum_prefix.codebook_logits).all())

        for invalid_kv_len in (0, CACHE_CAPACITY + 1):
            with self.subTest(kv_len=invalid_kv_len):
                with self.assertRaisesRegex(
                    ValueError,
                    f"kv_len must be between 1 and {CACHE_CAPACITY}",
                ):
                    model.forward_generate(
                        token,
                        input_pos=input_pos,
                        kv_len=invalid_kv_len,
                    )

        for invalid_input_pos, kv_len in ((-1, 1), (1, 1), (3, 3)):
            with self.subTest(input_pos=invalid_input_pos, kv_len=kv_len):
                with self.assertRaisesRegex(
                    RuntimeError,
                    "input_pos must be within the active KV prefix",
                ):
                    model.forward_generate(
                        token,
                        input_pos=torch.tensor([invalid_input_pos]),
                        kv_len=kv_len,
                    )

    def test_prefill_first_decode_and_maximum_prefix(self) -> None:
        model = self._model()
        original_sdpa = F.scaled_dot_product_attention
        observed_k_lengths = []

        def capture_sdpa(query, key, value, **kwargs):
            observed_k_lengths.append(
                (key.shape[-2], value.shape[-2], kwargs["attn_mask"].shape[-1])
            )
            return original_sdpa(query, key, value, **kwargs)

        with patch(
            "fish_speech.models.text2semantic.llama.F.scaled_dot_product_attention",
            side_effect=capture_sdpa,
        ):
            prefill = model.forward_generate(
                self._tokens(PROMPT_LENGTH),
                input_pos=torch.arange(PROMPT_LENGTH),
                kv_len=PROMPT_LENGTH,
            )
            first_decode = model.forward_generate(
                self._tokens(1),
                input_pos=torch.tensor([PROMPT_LENGTH]),
                kv_len=PROMPT_LENGTH + 1,
            )
            maximum_prefix = model.forward_generate(
                self._tokens(1),
                input_pos=torch.tensor([CACHE_CAPACITY - 1]),
                kv_len=CACHE_CAPACITY,
            )

        self.assertEqual(
            observed_k_lengths,
            [
                (PROMPT_LENGTH, PROMPT_LENGTH, PROMPT_LENGTH),
                (PROMPT_LENGTH + 1, PROMPT_LENGTH + 1, PROMPT_LENGTH + 1),
                (CACHE_CAPACITY, CACHE_CAPACITY, CACHE_CAPACITY),
            ],
        )
        for result in (prefill, first_decode, maximum_prefix):
            self.assertTrue(torch.isfinite(result.token_logits).all())
            self.assertTrue(torch.isfinite(result.codebook_logits).all())


class ActiveKVAutoregressiveDecodeTest(unittest.TestCase):
    @staticmethod
    def _model() -> DualARTransformer:
        torch.manual_seed(4)
        config = DualARModelArgs(
            **_base_args(
                n_fast_layer=1,
                fast_dim=8,
                fast_n_head=2,
                fast_n_local_heads=1,
                fast_head_dim=4,
                fast_intermediate_size=16,
            )
        )
        model = DualARTransformer(config).eval()
        model.tokenizer = _TokenizerStub()
        return model

    @staticmethod
    def _prompt() -> torch.Tensor:
        return torch.tensor(
            [
                [1, 2, 3],
                [0, 1, 2],
                [3, 4, 5],
            ],
            dtype=torch.long,
        )

    @staticmethod
    def _run_generation(model, *, use_active_prefix):
        slow_logits = []
        fast_logits = []
        observed_kv_lengths = []
        original_forward = model.forward_generate
        original_fast_forward = model.forward_generate_fast

        def capture_forward(*args, **kwargs):
            observed_kv_lengths.append(kwargs.get("kv_len"))
            if not use_active_prefix:
                kwargs["kv_len"] = None
            result = original_forward(*args, **kwargs)
            slow_logits.append(result.logits.detach().clone())
            result.logits = result.logits.clone()
            result.logits[..., model.tokenizer.eos_token_id] = -1e9
            return result

        def capture_fast_forward(*args, **kwargs):
            logits = original_fast_forward(*args, **kwargs)
            fast_logits.append(logits.detach().clone())
            return logits

        with (
            patch.object(model, "forward_generate", side_effect=capture_forward),
            patch.object(
                model,
                "forward_generate_fast",
                side_effect=capture_fast_forward,
            ),
            patch(
                "fish_speech.models.text2semantic.inference.tqdm",
                side_effect=lambda values, **_kwargs: values,
            ),
        ):
            output = generate(
                model=model,
                prompt=ActiveKVAutoregressiveDecodeTest._prompt(),
                max_new_tokens=3,
                audio_masks=None,
                audio_parts=None,
                temperature=0.8,
                top_p=0.8,
                top_k=1,
            )

        return output, slow_logits, fast_logits, observed_kv_lengths

    def test_real_decode_chain_preserves_greedy_tokens_and_distributions(self) -> None:
        active_model = self._model()
        full_cache_model = copy.deepcopy(active_model)

        torch.manual_seed(5)
        active = self._run_generation(active_model, use_active_prefix=True)
        torch.manual_seed(5)
        full = self._run_generation(full_cache_model, use_active_prefix=False)
        active_output, active_slow, active_fast, active_kv_lengths = active
        full_output, full_slow, full_fast, full_kv_lengths = full

        expected_shape = (1 + active_model.config.num_codebooks, PROMPT_LENGTH + 3)
        self.assertEqual(active_output.shape, expected_shape)
        self.assertEqual(full_output.shape, expected_shape)
        self.assertEqual(active_kv_lengths, [PROMPT_LENGTH, 4, 5])
        self.assertEqual(full_kv_lengths, active_kv_lengths)

        self.assertEqual(len(active_slow), len(full_slow))
        self.assertEqual(len(active_fast), len(full_fast))
        for active_logits, full_logits in zip(
            active_slow + active_fast,
            full_slow + full_fast,
        ):
            self.assertTrue(torch.isfinite(active_logits).all())
            self.assertTrue(torch.isfinite(full_logits).all())
            active_probs = torch.softmax(active_logits, dim=-1)
            full_probs = torch.softmax(full_logits, dim=-1)
            torch.testing.assert_close(active_probs, full_probs, rtol=1e-5, atol=1e-6)
            torch.testing.assert_close(
                active_probs.argmax(dim=-1),
                full_probs.argmax(dim=-1),
                rtol=0,
                atol=0,
            )

        torch.testing.assert_close(active_output, full_output, rtol=0, atol=0)

    def test_generated_codes_drop_only_a_closing_im_end(self) -> None:
        im_end = 9
        prompt = torch.zeros((3, 2), dtype=torch.long)
        for tail, frames in (([[5, im_end]], 1), ([[5, 6]], 2), ([[im_end]], 0)):
            main = torch.tensor(tail)
            y = torch.cat((prompt, torch.cat((main, main + 1, main + 2))), dim=1)
            codes = generated_codes(y, prompt_length=2, im_end_id=im_end)
            self.assertEqual(codes.shape, (2, frames))
            torch.testing.assert_close(codes, y[1:, 2 : 2 + frames])

    def test_max_new_tokens_one_and_unbounded(self) -> None:
        def run(max_new_tokens: int) -> torch.Tensor:
            with patch(
                "fish_speech.models.text2semantic.inference.tqdm",
                side_effect=lambda values, **_kwargs: values,
            ):
                return generate(
                    model=self._model(),
                    prompt=self._prompt(),
                    max_new_tokens=max_new_tokens,
                    audio_masks=None,
                    audio_parts=None,
                    temperature=0.8,
                    top_p=0.8,
                    top_k=1,
                )

        self.assertEqual(run(1).shape[1], PROMPT_LENGTH + 1)
        # Negative means no limit, like 0.
        torch.testing.assert_close(run(-1), run(0), rtol=0, atol=0)

    def test_legacy_decode_callback_remains_supported(self) -> None:
        model = self._model()

        def legacy_decode(
            model,
            x,
            input_pos,
            temperature,
            top_p,
            top_k,
            semantic_logit_bias,
            audio_masks,
            audio_parts,
            previous_tokens=None,
        ):
            return decode_one_token_ar(
                model=model,
                x=x,
                input_pos=input_pos,
                temperature=temperature,
                top_p=top_p,
                top_k=top_k,
                semantic_logit_bias=semantic_logit_bias,
                audio_masks=audio_masks,
                audio_parts=audio_parts,
                previous_tokens=previous_tokens,
            )

        with patch(
            "fish_speech.models.text2semantic.inference.tqdm",
            side_effect=lambda values, **_kwargs: values,
        ):
            output = generate(
                model=model,
                prompt=self._prompt(),
                max_new_tokens=2,
                audio_masks=None,
                audio_parts=None,
                temperature=0.8,
                top_p=0.8,
                top_k=1,
                decode_one_token=legacy_decode,
            )

        self.assertEqual(
            output.shape,
            (1 + model.config.num_codebooks, PROMPT_LENGTH + 2),
        )

    def test_compiled_decode_uses_one_dynamic_graph(self) -> None:
        model = self._model()
        model.output = _ConstantOutput(
            model.config.vocab_size,
            preferred_id=model.config.semantic_begin_id,
            suppressed_id=model.tokenizer.eos_token_id,
        )
        model.fast_output = _ConstantOutput(
            model.config.codebook_size,
            preferred_id=2,
        )
        compile_count = 0

        def counting_backend(graph_module, _example_inputs):
            nonlocal compile_count
            compile_count += 1
            return graph_module.forward

        torch._dynamo.reset()
        compiled_decode = torch.compile(
            decode_one_token_ar,
            backend=counting_backend,
            fullgraph=True,
            dynamic=True,
        )
        try:
            with patch(
                "fish_speech.models.text2semantic.inference.tqdm",
                side_effect=lambda values, **_kwargs: values,
            ):
                output = generate(
                    model=model,
                    prompt=self._prompt(),
                    max_new_tokens=4,
                    audio_masks=None,
                    audio_parts=None,
                    temperature=0.8,
                    top_p=0.8,
                    top_k=1,
                    decode_one_token=compiled_decode,
                )
        finally:
            torch._dynamo.reset()

        self.assertEqual(compile_count, 1)
        self.assertEqual(
            output.shape,
            (1 + model.config.num_codebooks, PROMPT_LENGTH + 4),
        )


if __name__ == "__main__":
    unittest.main()