SOTA Open Source TTS
perf(text2semantic): pass only active KV cache prefix to SDPA (#1312)
A.P.Liu214da3cd841b
diff --git a/fish_speech/models/text2semantic/inference.py b/fish_speech/models/text2semantic/inference.pyindex 63b977c..9fb82b0 100644--- a/fish_speech/models/text2semantic/inference.py+++ b/fish_speech/models/text2semantic/inference.py@@ -1,3 +1,4 @@+import inspectimport osimport queueimport re@@ -104,12 +105,14 @@ def decode_one_token_ar(audio_masks: torch.Tensor,audio_parts: torch.Tensor,previous_tokens: Optional[torch.Tensor] = None,+ kv_len: Optional[int] = None,) -> torch.Tensor:forward_result = model.forward_generate(x,input_pos,audio_masks=audio_masks,audio_parts=audio_parts,+ kv_len=kv_len,)logits = forward_result.logits # (1, 1, vocab_size)hidden_states = forward_result.hidden_states@@ -193,7 +196,13 @@ def decode_n_tokens(audio_masks: torch.Tensor,audio_parts: torch.Tensor,decode_one_token=decode_one_token_ar,+ kv_start_pos: Optional[int] = None,):+ if kv_start_pos is None:+ # Compatibility fallback for direct callers. The production generation+ # path passes the Python position explicitly to avoid a device sync.+ kv_start_pos = int(input_pos[0].item())+# Rolling window for RAS (Repetition Aware Sampling)previous_tokens = torch.zeros((model.config.num_codebooks + 1, RAS_WIN_SIZE),@@ -206,20 +215,30 @@ def decode_n_tokens(# [MODIFIED] Pre-fetch ID for efficiency loopim_end_id = model.tokenizer.get_token_id(IM_END_TOKEN)+ callback_parameters = inspect.signature(decode_one_token).parameters+ supports_kv_len = "kv_len" in callback_parameters or any(+ parameter.kind is inspect.Parameter.VAR_KEYWORD+ for parameter in callback_parameters.values()+ )+for i in tqdm(range(num_new_tokens)):+ decode_kwargs = {+ "model": model,+ "x": cur_token,+ "input_pos": input_pos,+ "previous_tokens": previous_tokens,+ "temperature": temperature,+ "top_p": top_p,+ "top_k": top_k,+ "semantic_logit_bias": semantic_logit_bias,+ "audio_masks": audio_masks,+ "audio_parts": audio_parts,+ }+ if supports_kv_len:+ decode_kwargs["kv_len"] = kv_start_pos + i + 1+with sdpa_kernel(SDPBackend.MATH):- next_token = decode_one_token(- model=model,- x=cur_token,- input_pos=input_pos,- previous_tokens=previous_tokens,- temperature=temperature,- top_p=top_p,- top_k=top_k,- semantic_logit_bias=semantic_logit_bias,- audio_masks=audio_masks,- audio_parts=audio_parts,- ).clone()+ next_token = decode_one_token(**decode_kwargs).clone()input_pos += 1cur_token = next_token.view(1, model.config.num_codebooks + 1, -1)@@ -331,7 +350,8 @@ def generate(semantic_logit_bias,audio_masks,audio_parts,- )+ kv_len=T,+ ).clone()seq[:, T : T + 1] = first_token# Recreate input_pos@@ -349,6 +369,7 @@ def generate(audio_masks=audio_masks,audio_parts=audio_parts,decode_one_token=decode_one_token,+ kv_start_pos=T,)seq = seq[:, : T + 1 + x.size(1)]seq[:, T + 1 :] = x@@ -387,6 +408,7 @@ def init_model(checkpoint_path, device, precision, compile=False):backend="inductor" if torch.cuda.is_available() else "aot_eager",mode="default" if torch.cuda.is_available() else None,fullgraph=True,+ dynamic=True,)return model.eval(), decode_one_tokendiff --git a/fish_speech/models/text2semantic/llama.py b/fish_speech/models/text2semantic/llama.pyindex c96aa46..648ce1a 100644--- a/fish_speech/models/text2semantic/llama.py+++ b/fish_speech/models/text2semantic/llama.py@@ -394,6 +394,7 @@ class BaseTransformer(nn.Module):audio_masks: Optional[Tensor] = None,audio_parts: Optional[Tensor] = None,return_all: bool = False,+ kv_len: Optional[int] = None,) -> BaseTransformerForwardResult:# Embedding logic replicated from embed() for compilation compatibility@@ -432,13 +433,32 @@ class BaseTransformer(nn.Module):else:logger.warning("audio_parts provided but model has no audio_projector")+ cache_capacity = (+ self.max_seq_len if self.max_seq_len > 0 else self.config.max_seq_len+ )if input_pos is None:input_pos = torch.arange(inp.shape[-1], device=x.device)- max_seq_len = inp.shape[-1]+ active_kv_len = inp.shape[-1]else:- max_seq_len = self.max_seq_len+ active_kv_len = cache_capacity if kv_len is None else kv_len++ if not 1 <= active_kv_len <= cache_capacity:+ raise ValueError(+ f"kv_len must be between 1 and {cache_capacity}, got {active_kv_len}"+ )++ input_pos_is_valid = ((input_pos >= 0) & (input_pos < active_kv_len)).all()+ if input_pos.device.type in {"cpu", "cuda"}:+ torch.ops.aten._assert_async.msg(+ input_pos_is_valid,+ "input_pos must be within the active KV prefix",+ )+ elif not torch.compiler.is_compiling() and not bool(input_pos_is_valid):+ raise ValueError("input_pos must be within the active KV prefix")- mask = self.causal_mask[None, None, input_pos, :max_seq_len] # (B, N, Q, K)+ mask = self.causal_mask[+ None, None, input_pos, :active_kv_len+ ] # (B, N, Q, active K)freqs_cis = self.freqs_cis[input_pos]for layer in self.layers:@@ -651,9 +671,12 @@ class NaiveTransformer(BaseTransformer):return self.decode(result)def forward_generate(- self, x: Tensor, input_pos: Optional[Tensor] = None+ self,+ x: Tensor,+ input_pos: Optional[Tensor] = None,+ kv_len: Optional[int] = None,) -> TransformerForwardResult:- result = super().forward_generate(x, input_pos)+ result = super().forward_generate(x, input_pos, kv_len=kv_len)return self.decode(result)@@ -822,8 +845,15 @@ class DualARTransformer(BaseTransformer):input_pos: Optional[Tensor] = None,audio_masks: Optional[Tensor] = None,audio_parts: Optional[Tensor] = None,+ kv_len: Optional[int] = None,) -> TransformerForwardResult:- x = super().forward_generate(x, input_pos, audio_masks, audio_parts)+ x = super().forward_generate(+ x,+ input_pos,+ audio_masks,+ audio_parts,+ kv_len=kv_len,+ )x.hidden_states = self.fast_project_in(x.hidden_states)return x@@ -909,6 +939,12 @@ class Attention(nn.Module):if self.kv_cache is not None:k, v = self.kv_cache.update(input_pos, k, v)+ if mask is not None:+ # Keep the cache allocated at max_seq_len, but repeat and+ # attend only to the prefix populated so far.+ active_kv_len = mask.shape[-1]+ k = k[:, :, :active_kv_len]+ v = v[:, :, :active_kv_len]k = k.repeat_interleave(self.n_head // self.n_local_heads, dim=1)v = v.repeat_interleave(self.n_head // self.n_local_heads, dim=1)diff --git a/tests/test_active_kv_attention.py b/tests/test_active_kv_attention.pynew file mode 100644index 0000000..1419d96--- /dev/null+++ b/tests/test_active_kv_attention.py@@ -0,0 +1,460 @@+import copy+import unittest+from unittest.mock import patch++import torch+from torch import nn+from torch.nn import functional as F++from fish_speech.models.text2semantic.inference import (+ decode_one_token_ar,+ generate,+)+from fish_speech.models.text2semantic.llama import (+ Attention,+ BaseModelArgs,+ DualARModelArgs,+ DualARTransformer,+ KVCache,+ NaiveModelArgs,+ NaiveTransformer,+ precompute_freqs_cis,+)++CACHE_CAPACITY = 8+PROMPT_LENGTH = 3+++class _TokenizerStub:+ eos_token_id = 15++ def get_token_id(self, _token: str) -> int:+ return self.eos_token_id+++class _ConstantOutput(nn.Module):+ def __init__(self, size: int, preferred_id: int, suppressed_id: int | None = None):+ super().__init__()+ logits = torch.zeros(size)+ logits[preferred_id] = 10.0+ if suppressed_id is not None:+ logits[suppressed_id] = -10.0+ self.register_buffer("logits", logits)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return self.logits.to(dtype=x.dtype).expand(*x.shape[:-1], -1)+++def _base_args(**overrides) -> dict:+ args = {+ "vocab_size": 16,+ "n_layer": 1,+ "n_head": 2,+ "n_local_heads": 1,+ "dim": 8,+ "intermediate_size": 16,+ "head_dim": 4,+ "max_seq_len": CACHE_CAPACITY,+ "dropout": 0.0,+ "tie_word_embeddings": False,+ "codebook_size": 8,+ "num_codebooks": 2,+ "semantic_begin_id": 4,+ "semantic_end_id": 11,+ "use_gradient_checkpointing": False,+ }+ args.update(overrides)+ return args+++class ActiveKVAttentionTest(unittest.TestCase):+ @staticmethod+ def _attention() -> Attention:+ config = BaseModelArgs(**_base_args())+ attention = Attention(config).eval()+ attention.kv_cache = KVCache(+ max_batch_size=1,+ max_seq_len=CACHE_CAPACITY,+ n_heads=1,+ head_dim=4,+ dtype=torch.float32,+ )+ return attention++ @staticmethod+ def _seed_cache(attention: Attention) -> None:+ torch.manual_seed(100)+ attention.kv_cache.k_cache[:, :, :2] = torch.randn(1, 1, 2, 4)+ attention.kv_cache.v_cache[:, :, :2] = torch.randn(1, 1, 2, 4)++ def test_attention_passes_only_active_kv_prefix_to_sdpa(self) -> None:+ torch.manual_seed(1)+ attention = self._attention()+ self._seed_cache(attention)+ x = torch.randn(1, 1, 8)+ freqs_cis = precompute_freqs_cis(CACHE_CAPACITY, 4)[2:3]+ mask = torch.ones(1, 1, 1, 3, dtype=torch.bool)+ original_sdpa = F.scaled_dot_product_attention+ observed_shapes = []++ def capture_sdpa(query, key, value, **kwargs):+ observed_shapes.append((key.shape, value.shape))+ return original_sdpa(query, key, value, **kwargs)++ with patch(+ "fish_speech.models.text2semantic.llama.F.scaled_dot_product_attention",+ side_effect=capture_sdpa,+ ):+ attention(x, freqs_cis, mask, input_pos=torch.tensor([2]))++ self.assertEqual(+ observed_shapes,+ [(torch.Size([1, 2, 3, 4]), torch.Size([1, 2, 3, 4]))],+ )+ self.assertEqual(attention.kv_cache.k_cache.shape[2], CACHE_CAPACITY)++ def test_active_prefix_matches_masked_full_cache_with_tolerance(self) -> None:+ torch.manual_seed(2)+ active_attention = self._attention()+ self._seed_cache(active_attention)+ full_attention = copy.deepcopy(active_attention)+ x = torch.randn(1, 1, 8)+ freqs_cis = precompute_freqs_cis(CACHE_CAPACITY, 4)[2:3]+ active_mask = torch.ones(1, 1, 1, 3, dtype=torch.bool)+ full_mask = torch.zeros(1, 1, 1, CACHE_CAPACITY, dtype=torch.bool)+ full_mask[..., :3] = True+ input_pos = torch.tensor([2])++ active_output = active_attention(+ x,+ freqs_cis,+ active_mask,+ input_pos=input_pos,+ )+ full_output = full_attention(+ x,+ freqs_cis,+ full_mask,+ input_pos=input_pos,+ )++ torch.testing.assert_close(active_output, full_output, rtol=1e-5, atol=1e-6)+ torch.testing.assert_close(+ active_output.argmax(dim=-1),+ full_output.argmax(dim=-1),+ rtol=0,+ atol=0,+ )+++class ActiveKVProgressionTest(unittest.TestCase):+ @staticmethod+ def _model() -> NaiveTransformer:+ torch.manual_seed(3)+ model = NaiveTransformer(NaiveModelArgs(**_base_args())).eval()+ model.setup_caches(1, CACHE_CAPACITY, dtype=torch.float32)+ return model++ @staticmethod+ def _tokens(length: int) -> torch.Tensor:+ main = torch.arange(1, length + 1).remainder(4).view(1, 1, length)+ codebooks = torch.arange(2 * length).remainder(8).view(1, 2, length)+ return torch.cat((main, codebooks), dim=1)++ def test_kv_len_validation(self) -> None:+ model = self._model()+ token = self._tokens(1)+ input_pos = torch.tensor([0])++ minimum_prefix = model.forward_generate(+ token,+ input_pos=input_pos,+ kv_len=1,+ )+ self.assertTrue(torch.isfinite(minimum_prefix.token_logits).all())+ self.assertTrue(torch.isfinite(minimum_prefix.codebook_logits).all())++ for invalid_kv_len in (0, CACHE_CAPACITY + 1):+ with self.subTest(kv_len=invalid_kv_len):+ with self.assertRaisesRegex(+ ValueError,+ f"kv_len must be between 1 and {CACHE_CAPACITY}",+ ):+ model.forward_generate(+ token,+ input_pos=input_pos,+ kv_len=invalid_kv_len,+ )++ for invalid_input_pos, kv_len in ((-1, 1), (1, 1), (3, 3)):+ with self.subTest(input_pos=invalid_input_pos, kv_len=kv_len):+ with self.assertRaisesRegex(+ RuntimeError,+ "input_pos must be within the active KV prefix",+ ):+ model.forward_generate(+ token,+ input_pos=torch.tensor([invalid_input_pos]),+ kv_len=kv_len,+ )++ def test_prefill_first_decode_and_maximum_prefix(self) -> None:+ model = self._model()+ original_sdpa = F.scaled_dot_product_attention+ observed_k_lengths = []++ def capture_sdpa(query, key, value, **kwargs):+ observed_k_lengths.append(+ (key.shape[-2], value.shape[-2], kwargs["attn_mask"].shape[-1])+ )+ return original_sdpa(query, key, value, **kwargs)++ with patch(+ "fish_speech.models.text2semantic.llama.F.scaled_dot_product_attention",+ side_effect=capture_sdpa,+ ):+ prefill = model.forward_generate(+ self._tokens(PROMPT_LENGTH),+ input_pos=torch.arange(PROMPT_LENGTH),+ kv_len=PROMPT_LENGTH,+ )+ first_decode = model.forward_generate(+ self._tokens(1),+ input_pos=torch.tensor([PROMPT_LENGTH]),+ kv_len=PROMPT_LENGTH + 1,+ )+ maximum_prefix = model.forward_generate(+ self._tokens(1),+ input_pos=torch.tensor([CACHE_CAPACITY - 1]),+ kv_len=CACHE_CAPACITY,+ )++ self.assertEqual(+ observed_k_lengths,+ [+ (PROMPT_LENGTH, PROMPT_LENGTH, PROMPT_LENGTH),+ (PROMPT_LENGTH + 1, PROMPT_LENGTH + 1, PROMPT_LENGTH + 1),+ (CACHE_CAPACITY, CACHE_CAPACITY, CACHE_CAPACITY),+ ],+ )+ for result in (prefill, first_decode, maximum_prefix):+ self.assertTrue(torch.isfinite(result.token_logits).all())+ self.assertTrue(torch.isfinite(result.codebook_logits).all())+++class ActiveKVAutoregressiveDecodeTest(unittest.TestCase):+ @staticmethod+ def _model() -> DualARTransformer:+ torch.manual_seed(4)+ config = DualARModelArgs(+ **_base_args(+ n_fast_layer=1,+ fast_dim=8,+ fast_n_head=2,+ fast_n_local_heads=1,+ fast_head_dim=4,+ fast_intermediate_size=16,+ )+ )+ model = DualARTransformer(config).eval()+ model.tokenizer = _TokenizerStub()+ return model++ @staticmethod+ def _prompt() -> torch.Tensor:+ return torch.tensor(+ [+ [1, 2, 3],+ [0, 1, 2],+ [3, 4, 5],+ ],+ dtype=torch.long,+ )++ @staticmethod+ def _run_generation(model, *, use_active_prefix):+ slow_logits = []+ fast_logits = []+ observed_kv_lengths = []+ original_forward = model.forward_generate+ original_fast_forward = model.forward_generate_fast++ def capture_forward(*args, **kwargs):+ observed_kv_lengths.append(kwargs.get("kv_len"))+ if not use_active_prefix:+ kwargs["kv_len"] = None+ result = original_forward(*args, **kwargs)+ slow_logits.append(result.logits.detach().clone())+ result.logits = result.logits.clone()+ result.logits[..., model.tokenizer.eos_token_id] = -1e9+ return result++ def capture_fast_forward(*args, **kwargs):+ logits = original_fast_forward(*args, **kwargs)+ fast_logits.append(logits.detach().clone())+ return logits++ with (+ patch.object(model, "forward_generate", side_effect=capture_forward),+ patch.object(+ model,+ "forward_generate_fast",+ side_effect=capture_fast_forward,+ ),+ patch(+ "fish_speech.models.text2semantic.inference.tqdm",+ side_effect=lambda values, **_kwargs: values,+ ),+ ):+ output = generate(+ model=model,+ prompt=ActiveKVAutoregressiveDecodeTest._prompt(),+ max_new_tokens=3,+ audio_masks=None,+ audio_parts=None,+ temperature=0.8,+ top_p=0.8,+ top_k=1,+ )++ return output, slow_logits, fast_logits, observed_kv_lengths++ def test_real_decode_chain_preserves_greedy_tokens_and_distributions(self) -> None:+ active_model = self._model()+ full_cache_model = copy.deepcopy(active_model)++ torch.manual_seed(5)+ active = self._run_generation(active_model, use_active_prefix=True)+ torch.manual_seed(5)+ full = self._run_generation(full_cache_model, use_active_prefix=False)+ active_output, active_slow, active_fast, active_kv_lengths = active+ full_output, full_slow, full_fast, full_kv_lengths = full++ expected_shape = (1 + active_model.config.num_codebooks, PROMPT_LENGTH + 3)+ self.assertEqual(active_output.shape, expected_shape)+ self.assertEqual(full_output.shape, expected_shape)+ self.assertEqual(active_kv_lengths, [PROMPT_LENGTH, 4, 5])+ self.assertEqual(full_kv_lengths, active_kv_lengths)++ self.assertEqual(len(active_slow), len(full_slow))+ self.assertEqual(len(active_fast), len(full_fast))+ for active_logits, full_logits in zip(+ active_slow + active_fast,+ full_slow + full_fast,+ ):+ self.assertTrue(torch.isfinite(active_logits).all())+ self.assertTrue(torch.isfinite(full_logits).all())+ active_probs = torch.softmax(active_logits, dim=-1)+ full_probs = torch.softmax(full_logits, dim=-1)+ torch.testing.assert_close(active_probs, full_probs, rtol=1e-5, atol=1e-6)+ torch.testing.assert_close(+ active_probs.argmax(dim=-1),+ full_probs.argmax(dim=-1),+ rtol=0,+ atol=0,+ )++ torch.testing.assert_close(active_output, full_output, rtol=0, atol=0)++ def test_legacy_decode_callback_remains_supported(self) -> None:+ model = self._model()++ def legacy_decode(+ model,+ x,+ input_pos,+ temperature,+ top_p,+ top_k,+ semantic_logit_bias,+ audio_masks,+ audio_parts,+ previous_tokens=None,+ ):+ return decode_one_token_ar(+ model=model,+ x=x,+ input_pos=input_pos,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ semantic_logit_bias=semantic_logit_bias,+ audio_masks=audio_masks,+ audio_parts=audio_parts,+ previous_tokens=previous_tokens,+ )++ with patch(+ "fish_speech.models.text2semantic.inference.tqdm",+ side_effect=lambda values, **_kwargs: values,+ ):+ output = generate(+ model=model,+ prompt=self._prompt(),+ max_new_tokens=2,+ audio_masks=None,+ audio_parts=None,+ temperature=0.8,+ top_p=0.8,+ top_k=1,+ decode_one_token=legacy_decode,+ )++ self.assertEqual(+ output.shape,+ (1 + model.config.num_codebooks, PROMPT_LENGTH + 2),+ )++ def test_compiled_decode_uses_one_dynamic_graph(self) -> None:+ model = self._model()+ model.output = _ConstantOutput(+ model.config.vocab_size,+ preferred_id=model.config.semantic_begin_id,+ suppressed_id=model.tokenizer.eos_token_id,+ )+ model.fast_output = _ConstantOutput(+ model.config.codebook_size,+ preferred_id=2,+ )+ compile_count = 0++ def counting_backend(graph_module, _example_inputs):+ nonlocal compile_count+ compile_count += 1+ return graph_module.forward++ torch._dynamo.reset()+ compiled_decode = torch.compile(+ decode_one_token_ar,+ backend=counting_backend,+ fullgraph=True,+ dynamic=True,+ )+ try:+ with patch(+ "fish_speech.models.text2semantic.inference.tqdm",+ side_effect=lambda values, **_kwargs: values,+ ):+ output = generate(+ model=model,+ prompt=self._prompt(),+ max_new_tokens=4,+ audio_masks=None,+ audio_parts=None,+ temperature=0.8,+ top_p=0.8,+ top_k=1,+ decode_one_token=compiled_decode,+ )+ finally:+ torch._dynamo.reset()++ self.assertEqual(compile_count, 1)+ self.assertEqual(+ output.shape,+ (1 + model.config.num_codebooks, PROMPT_LENGTH + 4),+ )+++if __name__ == "__main__":+ unittest.main()