Skip to content

SOTA Open Source TTS

perf(text2semantic): pass only active KV cache prefix to SDPA (#1312)

A.P.Liu214da3cd841b
diff --git a/fish_speech/models/text2semantic/inference.py b/fish_speech/models/text2semantic/inference.py
index 63b977c..9fb82b0 100644
--- a/fish_speech/models/text2semantic/inference.py
+++ b/fish_speech/models/text2semantic/inference.py
@@ -1,3 +1,4 @@
+import inspect
import os
import queue
import re
@@ -104,12 +105,14 @@ def decode_one_token_ar(
audio_masks: torch.Tensor,
audio_parts: torch.Tensor,
previous_tokens: Optional[torch.Tensor] = None,
+ kv_len: Optional[int] = None,
) -> torch.Tensor:
forward_result = model.forward_generate(
x,
input_pos,
audio_masks=audio_masks,
audio_parts=audio_parts,
+ kv_len=kv_len,
)
logits = forward_result.logits # (1, 1, vocab_size)
hidden_states = forward_result.hidden_states
@@ -193,7 +196,13 @@ def decode_n_tokens(
audio_masks: torch.Tensor,
audio_parts: torch.Tensor,
decode_one_token=decode_one_token_ar,
+ kv_start_pos: Optional[int] = None,
):
+ if kv_start_pos is None:
+ # Compatibility fallback for direct callers. The production generation
+ # path passes the Python position explicitly to avoid a device sync.
+ kv_start_pos = int(input_pos[0].item())
+
# Rolling window for RAS (Repetition Aware Sampling)
previous_tokens = torch.zeros(
(model.config.num_codebooks + 1, RAS_WIN_SIZE),
@@ -206,20 +215,30 @@ def decode_n_tokens(
# [MODIFIED] Pre-fetch ID for efficiency loop
im_end_id = model.tokenizer.get_token_id(IM_END_TOKEN)
+ callback_parameters = inspect.signature(decode_one_token).parameters
+ supports_kv_len = "kv_len" in callback_parameters or any(
+ parameter.kind is inspect.Parameter.VAR_KEYWORD
+ for parameter in callback_parameters.values()
+ )
+
for i in tqdm(range(num_new_tokens)):
+ decode_kwargs = {
+ "model": model,
+ "x": cur_token,
+ "input_pos": input_pos,
+ "previous_tokens": previous_tokens,
+ "temperature": temperature,
+ "top_p": top_p,
+ "top_k": top_k,
+ "semantic_logit_bias": semantic_logit_bias,
+ "audio_masks": audio_masks,
+ "audio_parts": audio_parts,
+ }
+ if supports_kv_len:
+ decode_kwargs["kv_len"] = kv_start_pos + i + 1
+
with sdpa_kernel(SDPBackend.MATH):
- next_token = decode_one_token(
- model=model,
- x=cur_token,
- input_pos=input_pos,
- previous_tokens=previous_tokens,
- temperature=temperature,
- top_p=top_p,
- top_k=top_k,
- semantic_logit_bias=semantic_logit_bias,
- audio_masks=audio_masks,
- audio_parts=audio_parts,
- ).clone()
+ next_token = decode_one_token(**decode_kwargs).clone()
input_pos += 1
cur_token = next_token.view(1, model.config.num_codebooks + 1, -1)
@@ -331,7 +350,8 @@ def generate(
semantic_logit_bias,
audio_masks,
audio_parts,
- )
+ kv_len=T,
+ ).clone()
seq[:, T : T + 1] = first_token
# Recreate input_pos
@@ -349,6 +369,7 @@ def generate(
audio_masks=audio_masks,
audio_parts=audio_parts,
decode_one_token=decode_one_token,
+ kv_start_pos=T,
)
seq = seq[:, : T + 1 + x.size(1)]
seq[:, T + 1 :] = x
@@ -387,6 +408,7 @@ def init_model(checkpoint_path, device, precision, compile=False):
backend="inductor" if torch.cuda.is_available() else "aot_eager",
mode="default" if torch.cuda.is_available() else None,
fullgraph=True,
+ dynamic=True,
)
return model.eval(), decode_one_token
diff --git a/fish_speech/models/text2semantic/llama.py b/fish_speech/models/text2semantic/llama.py
index c96aa46..648ce1a 100644
--- a/fish_speech/models/text2semantic/llama.py
+++ b/fish_speech/models/text2semantic/llama.py
@@ -394,6 +394,7 @@ class BaseTransformer(nn.Module):
audio_masks: Optional[Tensor] = None,
audio_parts: Optional[Tensor] = None,
return_all: bool = False,
+ kv_len: Optional[int] = None,
) -> BaseTransformerForwardResult:
# Embedding logic replicated from embed() for compilation compatibility
@@ -432,13 +433,32 @@ class BaseTransformer(nn.Module):
else:
logger.warning("audio_parts provided but model has no audio_projector")
+ cache_capacity = (
+ self.max_seq_len if self.max_seq_len > 0 else self.config.max_seq_len
+ )
if input_pos is None:
input_pos = torch.arange(inp.shape[-1], device=x.device)
- max_seq_len = inp.shape[-1]
+ active_kv_len = inp.shape[-1]
else:
- max_seq_len = self.max_seq_len
+ active_kv_len = cache_capacity if kv_len is None else kv_len
+
+ if not 1 <= active_kv_len <= cache_capacity:
+ raise ValueError(
+ f"kv_len must be between 1 and {cache_capacity}, got {active_kv_len}"
+ )
+
+ input_pos_is_valid = ((input_pos >= 0) & (input_pos < active_kv_len)).all()
+ if input_pos.device.type in {"cpu", "cuda"}:
+ torch.ops.aten._assert_async.msg(
+ input_pos_is_valid,
+ "input_pos must be within the active KV prefix",
+ )
+ elif not torch.compiler.is_compiling() and not bool(input_pos_is_valid):
+ raise ValueError("input_pos must be within the active KV prefix")
- mask = self.causal_mask[None, None, input_pos, :max_seq_len] # (B, N, Q, K)
+ mask = self.causal_mask[
+ None, None, input_pos, :active_kv_len
+ ] # (B, N, Q, active K)
freqs_cis = self.freqs_cis[input_pos]
for layer in self.layers:
@@ -651,9 +671,12 @@ class NaiveTransformer(BaseTransformer):
return self.decode(result)
def forward_generate(
- self, x: Tensor, input_pos: Optional[Tensor] = None
+ self,
+ x: Tensor,
+ input_pos: Optional[Tensor] = None,
+ kv_len: Optional[int] = None,
) -> TransformerForwardResult:
- result = super().forward_generate(x, input_pos)
+ result = super().forward_generate(x, input_pos, kv_len=kv_len)
return self.decode(result)
@@ -822,8 +845,15 @@ class DualARTransformer(BaseTransformer):
input_pos: Optional[Tensor] = None,
audio_masks: Optional[Tensor] = None,
audio_parts: Optional[Tensor] = None,
+ kv_len: Optional[int] = None,
) -> TransformerForwardResult:
- x = super().forward_generate(x, input_pos, audio_masks, audio_parts)
+ x = super().forward_generate(
+ x,
+ input_pos,
+ audio_masks,
+ audio_parts,
+ kv_len=kv_len,
+ )
x.hidden_states = self.fast_project_in(x.hidden_states)
return x
@@ -909,6 +939,12 @@ class Attention(nn.Module):
if self.kv_cache is not None:
k, v = self.kv_cache.update(input_pos, k, v)
+ if mask is not None:
+ # Keep the cache allocated at max_seq_len, but repeat and
+ # attend only to the prefix populated so far.
+ active_kv_len = mask.shape[-1]
+ k = k[:, :, :active_kv_len]
+ v = v[:, :, :active_kv_len]
k = k.repeat_interleave(self.n_head // self.n_local_heads, dim=1)
v = v.repeat_interleave(self.n_head // self.n_local_heads, dim=1)
diff --git a/tests/test_active_kv_attention.py b/tests/test_active_kv_attention.py
new file mode 100644
index 0000000..1419d96
--- /dev/null
+++ b/tests/test_active_kv_attention.py
@@ -0,0 +1,460 @@
+import copy
+import unittest
+from unittest.mock import patch
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from fish_speech.models.text2semantic.inference import (
+ decode_one_token_ar,
+ generate,
+)
+from fish_speech.models.text2semantic.llama import (
+ Attention,
+ BaseModelArgs,
+ DualARModelArgs,
+ DualARTransformer,
+ KVCache,
+ NaiveModelArgs,
+ NaiveTransformer,
+ precompute_freqs_cis,
+)
+
+CACHE_CAPACITY = 8
+PROMPT_LENGTH = 3
+
+
+class _TokenizerStub:
+ eos_token_id = 15
+
+ def get_token_id(self, _token: str) -> int:
+ return self.eos_token_id
+
+
+class _ConstantOutput(nn.Module):
+ def __init__(self, size: int, preferred_id: int, suppressed_id: int | None = None):
+ super().__init__()
+ logits = torch.zeros(size)
+ logits[preferred_id] = 10.0
+ if suppressed_id is not None:
+ logits[suppressed_id] = -10.0
+ self.register_buffer("logits", logits)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return self.logits.to(dtype=x.dtype).expand(*x.shape[:-1], -1)
+
+
+def _base_args(**overrides) -> dict:
+ args = {
+ "vocab_size": 16,
+ "n_layer": 1,
+ "n_head": 2,
+ "n_local_heads": 1,
+ "dim": 8,
+ "intermediate_size": 16,
+ "head_dim": 4,
+ "max_seq_len": CACHE_CAPACITY,
+ "dropout": 0.0,
+ "tie_word_embeddings": False,
+ "codebook_size": 8,
+ "num_codebooks": 2,
+ "semantic_begin_id": 4,
+ "semantic_end_id": 11,
+ "use_gradient_checkpointing": False,
+ }
+ args.update(overrides)
+ return args
+
+
+class ActiveKVAttentionTest(unittest.TestCase):
+ @staticmethod
+ def _attention() -> Attention:
+ config = BaseModelArgs(**_base_args())
+ attention = Attention(config).eval()
+ attention.kv_cache = KVCache(
+ max_batch_size=1,
+ max_seq_len=CACHE_CAPACITY,
+ n_heads=1,
+ head_dim=4,
+ dtype=torch.float32,
+ )
+ return attention
+
+ @staticmethod
+ def _seed_cache(attention: Attention) -> None:
+ torch.manual_seed(100)
+ attention.kv_cache.k_cache[:, :, :2] = torch.randn(1, 1, 2, 4)
+ attention.kv_cache.v_cache[:, :, :2] = torch.randn(1, 1, 2, 4)
+
+ def test_attention_passes_only_active_kv_prefix_to_sdpa(self) -> None:
+ torch.manual_seed(1)
+ attention = self._attention()
+ self._seed_cache(attention)
+ x = torch.randn(1, 1, 8)
+ freqs_cis = precompute_freqs_cis(CACHE_CAPACITY, 4)[2:3]
+ mask = torch.ones(1, 1, 1, 3, dtype=torch.bool)
+ original_sdpa = F.scaled_dot_product_attention
+ observed_shapes = []
+
+ def capture_sdpa(query, key, value, **kwargs):
+ observed_shapes.append((key.shape, value.shape))
+ return original_sdpa(query, key, value, **kwargs)
+
+ with patch(
+ "fish_speech.models.text2semantic.llama.F.scaled_dot_product_attention",
+ side_effect=capture_sdpa,
+ ):
+ attention(x, freqs_cis, mask, input_pos=torch.tensor([2]))
+
+ self.assertEqual(
+ observed_shapes,
+ [(torch.Size([1, 2, 3, 4]), torch.Size([1, 2, 3, 4]))],
+ )
+ self.assertEqual(attention.kv_cache.k_cache.shape[2], CACHE_CAPACITY)
+
+ def test_active_prefix_matches_masked_full_cache_with_tolerance(self) -> None:
+ torch.manual_seed(2)
+ active_attention = self._attention()
+ self._seed_cache(active_attention)
+ full_attention = copy.deepcopy(active_attention)
+ x = torch.randn(1, 1, 8)
+ freqs_cis = precompute_freqs_cis(CACHE_CAPACITY, 4)[2:3]
+ active_mask = torch.ones(1, 1, 1, 3, dtype=torch.bool)
+ full_mask = torch.zeros(1, 1, 1, CACHE_CAPACITY, dtype=torch.bool)
+ full_mask[..., :3] = True
+ input_pos = torch.tensor([2])
+
+ active_output = active_attention(
+ x,
+ freqs_cis,
+ active_mask,
+ input_pos=input_pos,
+ )
+ full_output = full_attention(
+ x,
+ freqs_cis,
+ full_mask,
+ input_pos=input_pos,
+ )
+
+ torch.testing.assert_close(active_output, full_output, rtol=1e-5, atol=1e-6)
+ torch.testing.assert_close(
+ active_output.argmax(dim=-1),
+ full_output.argmax(dim=-1),
+ rtol=0,
+ atol=0,
+ )
+
+
+class ActiveKVProgressionTest(unittest.TestCase):
+ @staticmethod
+ def _model() -> NaiveTransformer:
+ torch.manual_seed(3)
+ model = NaiveTransformer(NaiveModelArgs(**_base_args())).eval()
+ model.setup_caches(1, CACHE_CAPACITY, dtype=torch.float32)
+ return model
+
+ @staticmethod
+ def _tokens(length: int) -> torch.Tensor:
+ main = torch.arange(1, length + 1).remainder(4).view(1, 1, length)
+ codebooks = torch.arange(2 * length).remainder(8).view(1, 2, length)
+ return torch.cat((main, codebooks), dim=1)
+
+ def test_kv_len_validation(self) -> None:
+ model = self._model()
+ token = self._tokens(1)
+ input_pos = torch.tensor([0])
+
+ minimum_prefix = model.forward_generate(
+ token,
+ input_pos=input_pos,
+ kv_len=1,
+ )
+ self.assertTrue(torch.isfinite(minimum_prefix.token_logits).all())
+ self.assertTrue(torch.isfinite(minimum_prefix.codebook_logits).all())
+
+ for invalid_kv_len in (0, CACHE_CAPACITY + 1):
+ with self.subTest(kv_len=invalid_kv_len):
+ with self.assertRaisesRegex(
+ ValueError,
+ f"kv_len must be between 1 and {CACHE_CAPACITY}",
+ ):
+ model.forward_generate(
+ token,
+ input_pos=input_pos,
+ kv_len=invalid_kv_len,
+ )
+
+ for invalid_input_pos, kv_len in ((-1, 1), (1, 1), (3, 3)):
+ with self.subTest(input_pos=invalid_input_pos, kv_len=kv_len):
+ with self.assertRaisesRegex(
+ RuntimeError,
+ "input_pos must be within the active KV prefix",
+ ):
+ model.forward_generate(
+ token,
+ input_pos=torch.tensor([invalid_input_pos]),
+ kv_len=kv_len,
+ )
+
+ def test_prefill_first_decode_and_maximum_prefix(self) -> None:
+ model = self._model()
+ original_sdpa = F.scaled_dot_product_attention
+ observed_k_lengths = []
+
+ def capture_sdpa(query, key, value, **kwargs):
+ observed_k_lengths.append(
+ (key.shape[-2], value.shape[-2], kwargs["attn_mask"].shape[-1])
+ )
+ return original_sdpa(query, key, value, **kwargs)
+
+ with patch(
+ "fish_speech.models.text2semantic.llama.F.scaled_dot_product_attention",
+ side_effect=capture_sdpa,
+ ):
+ prefill = model.forward_generate(
+ self._tokens(PROMPT_LENGTH),
+ input_pos=torch.arange(PROMPT_LENGTH),
+ kv_len=PROMPT_LENGTH,
+ )
+ first_decode = model.forward_generate(
+ self._tokens(1),
+ input_pos=torch.tensor([PROMPT_LENGTH]),
+ kv_len=PROMPT_LENGTH + 1,
+ )
+ maximum_prefix = model.forward_generate(
+ self._tokens(1),
+ input_pos=torch.tensor([CACHE_CAPACITY - 1]),
+ kv_len=CACHE_CAPACITY,
+ )
+
+ self.assertEqual(
+ observed_k_lengths,
+ [
+ (PROMPT_LENGTH, PROMPT_LENGTH, PROMPT_LENGTH),
+ (PROMPT_LENGTH + 1, PROMPT_LENGTH + 1, PROMPT_LENGTH + 1),
+ (CACHE_CAPACITY, CACHE_CAPACITY, CACHE_CAPACITY),
+ ],
+ )
+ for result in (prefill, first_decode, maximum_prefix):
+ self.assertTrue(torch.isfinite(result.token_logits).all())
+ self.assertTrue(torch.isfinite(result.codebook_logits).all())
+
+
+class ActiveKVAutoregressiveDecodeTest(unittest.TestCase):
+ @staticmethod
+ def _model() -> DualARTransformer:
+ torch.manual_seed(4)
+ config = DualARModelArgs(
+ **_base_args(
+ n_fast_layer=1,
+ fast_dim=8,
+ fast_n_head=2,
+ fast_n_local_heads=1,
+ fast_head_dim=4,
+ fast_intermediate_size=16,
+ )
+ )
+ model = DualARTransformer(config).eval()
+ model.tokenizer = _TokenizerStub()
+ return model
+
+ @staticmethod
+ def _prompt() -> torch.Tensor:
+ return torch.tensor(
+ [
+ [1, 2, 3],
+ [0, 1, 2],
+ [3, 4, 5],
+ ],
+ dtype=torch.long,
+ )
+
+ @staticmethod
+ def _run_generation(model, *, use_active_prefix):
+ slow_logits = []
+ fast_logits = []
+ observed_kv_lengths = []
+ original_forward = model.forward_generate
+ original_fast_forward = model.forward_generate_fast
+
+ def capture_forward(*args, **kwargs):
+ observed_kv_lengths.append(kwargs.get("kv_len"))
+ if not use_active_prefix:
+ kwargs["kv_len"] = None
+ result = original_forward(*args, **kwargs)
+ slow_logits.append(result.logits.detach().clone())
+ result.logits = result.logits.clone()
+ result.logits[..., model.tokenizer.eos_token_id] = -1e9
+ return result
+
+ def capture_fast_forward(*args, **kwargs):
+ logits = original_fast_forward(*args, **kwargs)
+ fast_logits.append(logits.detach().clone())
+ return logits
+
+ with (
+ patch.object(model, "forward_generate", side_effect=capture_forward),
+ patch.object(
+ model,
+ "forward_generate_fast",
+ side_effect=capture_fast_forward,
+ ),
+ patch(
+ "fish_speech.models.text2semantic.inference.tqdm",
+ side_effect=lambda values, **_kwargs: values,
+ ),
+ ):
+ output = generate(
+ model=model,
+ prompt=ActiveKVAutoregressiveDecodeTest._prompt(),
+ max_new_tokens=3,
+ audio_masks=None,
+ audio_parts=None,
+ temperature=0.8,
+ top_p=0.8,
+ top_k=1,
+ )
+
+ return output, slow_logits, fast_logits, observed_kv_lengths
+
+ def test_real_decode_chain_preserves_greedy_tokens_and_distributions(self) -> None:
+ active_model = self._model()
+ full_cache_model = copy.deepcopy(active_model)
+
+ torch.manual_seed(5)
+ active = self._run_generation(active_model, use_active_prefix=True)
+ torch.manual_seed(5)
+ full = self._run_generation(full_cache_model, use_active_prefix=False)
+ active_output, active_slow, active_fast, active_kv_lengths = active
+ full_output, full_slow, full_fast, full_kv_lengths = full
+
+ expected_shape = (1 + active_model.config.num_codebooks, PROMPT_LENGTH + 3)
+ self.assertEqual(active_output.shape, expected_shape)
+ self.assertEqual(full_output.shape, expected_shape)
+ self.assertEqual(active_kv_lengths, [PROMPT_LENGTH, 4, 5])
+ self.assertEqual(full_kv_lengths, active_kv_lengths)
+
+ self.assertEqual(len(active_slow), len(full_slow))
+ self.assertEqual(len(active_fast), len(full_fast))
+ for active_logits, full_logits in zip(
+ active_slow + active_fast,
+ full_slow + full_fast,
+ ):
+ self.assertTrue(torch.isfinite(active_logits).all())
+ self.assertTrue(torch.isfinite(full_logits).all())
+ active_probs = torch.softmax(active_logits, dim=-1)
+ full_probs = torch.softmax(full_logits, dim=-1)
+ torch.testing.assert_close(active_probs, full_probs, rtol=1e-5, atol=1e-6)
+ torch.testing.assert_close(
+ active_probs.argmax(dim=-1),
+ full_probs.argmax(dim=-1),
+ rtol=0,
+ atol=0,
+ )
+
+ torch.testing.assert_close(active_output, full_output, rtol=0, atol=0)
+
+ def test_legacy_decode_callback_remains_supported(self) -> None:
+ model = self._model()
+
+ def legacy_decode(
+ model,
+ x,
+ input_pos,
+ temperature,
+ top_p,
+ top_k,
+ semantic_logit_bias,
+ audio_masks,
+ audio_parts,
+ previous_tokens=None,
+ ):
+ return decode_one_token_ar(
+ model=model,
+ x=x,
+ input_pos=input_pos,
+ temperature=temperature,
+ top_p=top_p,
+ top_k=top_k,
+ semantic_logit_bias=semantic_logit_bias,
+ audio_masks=audio_masks,
+ audio_parts=audio_parts,
+ previous_tokens=previous_tokens,
+ )
+
+ with patch(
+ "fish_speech.models.text2semantic.inference.tqdm",
+ side_effect=lambda values, **_kwargs: values,
+ ):
+ output = generate(
+ model=model,
+ prompt=self._prompt(),
+ max_new_tokens=2,
+ audio_masks=None,
+ audio_parts=None,
+ temperature=0.8,
+ top_p=0.8,
+ top_k=1,
+ decode_one_token=legacy_decode,
+ )
+
+ self.assertEqual(
+ output.shape,
+ (1 + model.config.num_codebooks, PROMPT_LENGTH + 2),
+ )
+
+ def test_compiled_decode_uses_one_dynamic_graph(self) -> None:
+ model = self._model()
+ model.output = _ConstantOutput(
+ model.config.vocab_size,
+ preferred_id=model.config.semantic_begin_id,
+ suppressed_id=model.tokenizer.eos_token_id,
+ )
+ model.fast_output = _ConstantOutput(
+ model.config.codebook_size,
+ preferred_id=2,
+ )
+ compile_count = 0
+
+ def counting_backend(graph_module, _example_inputs):
+ nonlocal compile_count
+ compile_count += 1
+ return graph_module.forward
+
+ torch._dynamo.reset()
+ compiled_decode = torch.compile(
+ decode_one_token_ar,
+ backend=counting_backend,
+ fullgraph=True,
+ dynamic=True,
+ )
+ try:
+ with patch(
+ "fish_speech.models.text2semantic.inference.tqdm",
+ side_effect=lambda values, **_kwargs: values,
+ ):
+ output = generate(
+ model=model,
+ prompt=self._prompt(),
+ max_new_tokens=4,
+ audio_masks=None,
+ audio_parts=None,
+ temperature=0.8,
+ top_p=0.8,
+ top_k=1,
+ decode_one_token=compiled_decode,
+ )
+ finally:
+ torch._dynamo.reset()
+
+ self.assertEqual(compile_count, 1)
+ self.assertEqual(
+ output.shape,
+ (1 + model.config.num_codebooks, PROMPT_LENGTH + 4),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()