SOTA Open Source TTS
fix: honor conversation visualization options (#1331)
Avinash Gola44a034dc3622
diff --git a/fish_speech/content_sequence.py b/fish_speech/content_sequence.pyindex c29f8c0..8ec8136 100644--- a/fish_speech/content_sequence.py+++ b/fish_speech/content_sequence.py@@ -5,6 +5,7 @@ import numpy as npimport torchfrom fish_speech.tokenizer import (+ AUDIO_EMBED_TOKEN,IM_END_TOKEN,MODALITY_TOKENS,FishTokenizer,@@ -328,6 +329,8 @@ class ContentSequence:tokenizer: FishTokenizer,ignore_loss_tokens: list[str] = [],merge_semantic_tokens: bool = False,+ merge_audio_tokens: bool = False,+ use_color: bool = True,):"""Visualize the encoded sequence with color-coded tokens.@@ -349,41 +352,69 @@ class ContentSequence:def print_in_blue(x):nonlocal blue_idx- color = colors["blue"] if blue_idx % 2 == 0 else colors["cyan"]- print(f"{color}{x}\033[0m", end="")+ if use_color:+ color = colors["blue"] if blue_idx % 2 == 0 else colors["cyan"]+ print(f"{color}{x}\033[0m", end="")+ else:+ print(x, end="")blue_idx += 1def print_in_green(x):nonlocal green_idx- color = colors["green"] if green_idx % 2 == 0 else colors["dark_green"]- print(f"{color}{x}\033[0m", end="")+ if use_color:+ color = colors["green"] if green_idx % 2 == 0 else colors["dark_green"]+ print(f"{color}{x}\033[0m", end="")+ else:+ print(x, end="")green_idx += 1- def print_semantic_token(x, count):- val = f"[<|semantic|>x{count}]"- if x == -100:+ def print_merged_token(name: str, label: int, count: int):+ val = f"[<|{name}|>x{count}]"+ if label == -100:print_in_green(val)else:print_in_blue(val)- count_semantic_tokens = 0- semantic_label = None+ audio_token_id = (+ tokenizer.get_token_id(AUDIO_EMBED_TOKEN) if merge_audio_tokens else None+ )+ merged_kind = None+ merged_label = None+ merged_count = 0++ def flush_merged_tokens():+ nonlocal merged_kind, merged_label, merged_count+ if merged_kind is not None:+ assert merged_label is not None+ print_merged_token(merged_kind, merged_label, merged_count)+ merged_kind = None+ merged_label = None+ merged_count = 0for tok, lab in zip(encoded.tokens, encoded.labels):token_id = int(tok.item())+ label = int(lab.item())++ token_kind = None+ if (+ merge_semantic_tokens+ and tokenizer.semantic_begin_id <= token_id <= tokenizer.semantic_end_id+ ):+ token_kind = "semantic"+ elif merge_audio_tokens and token_id == audio_token_id:+ token_kind = "audio_pad"++ if token_kind is not None:+ if token_kind == merged_kind and label == merged_label:+ merged_count += 1+ else:+ flush_merged_tokens()+ merged_kind = token_kind+ merged_label = label+ merged_count = 1+ continue- if merge_semantic_tokens:- if (- tokenizer.semantic_begin_id <= token_id <= tokenizer.semantic_end_id- and (semantic_label is None or semantic_label == lab)- ):- count_semantic_tokens += 1- semantic_label = lab- continue- elif count_semantic_tokens > 0:- print_semantic_token(semantic_label, count_semantic_tokens)- count_semantic_tokens = 0- semantic_label = None+ flush_merged_tokens()# Use HF decodeval = tokenizer.decode([token_id])@@ -392,12 +423,11 @@ class ContentSequence:if not val:val = f"<{token_id}>"- if lab == -100:+ if label == -100:print_in_green(val)else:print_in_blue(val)- if merge_semantic_tokens and count_semantic_tokens > 0:- print_semantic_token(semantic_label, count_semantic_tokens)+ flush_merged_tokens()print()diff --git a/fish_speech/conversation.py b/fish_speech/conversation.pyindex d0fa5c2..1629202 100644--- a/fish_speech/conversation.py+++ b/fish_speech/conversation.py@@ -120,6 +120,8 @@ class Conversation:tokenizer,ignore_loss_tokens=ignore_loss_tokens,merge_semantic_tokens=merge_semantic_tokens,+ merge_audio_tokens=merge_audio_tokens,+ use_color=use_color,)def append(self: "Conversation", message: Message):diff --git a/tests/test_visualization.py b/tests/test_visualization.pynew file mode 100644index 0000000..e70dd2d--- /dev/null+++ b/tests/test_visualization.py@@ -0,0 +1,95 @@+from types import SimpleNamespace++from fish_speech.content_sequence import ContentSequence+from fish_speech.conversation import Conversation+from fish_speech.tokenizer import AUDIO_EMBED_TOKEN+++class FakeTokenizer:+ semantic_begin_id = 100+ semantic_end_id = 199+ audio_token_id = 200++ def get_token_id(self, token):+ assert token == AUDIO_EMBED_TOKEN+ return self.audio_token_id++ def decode(self, tokens):+ return {+ 1: "hello",+ self.audio_token_id: AUDIO_EMBED_TOKEN,+ }.get(tokens[0], f"<|semantic:{tokens[0] - self.semantic_begin_id}|>")+++class FakeScalar(int):+ def item(self):+ return int(self)+++def encoded(tokens, labels):+ return SimpleNamespace(+ tokens=[FakeScalar(token) for token in tokens],+ labels=[FakeScalar(label) for label in labels],+ )+++def test_content_sequence_visualize_without_color(monkeypatch, capsys):+ sequence = ContentSequence()+ monkeypatch.setattr(+ sequence, "encode", lambda *args, **kwargs: encoded([1], [-100])+ )++ sequence.visualize(FakeTokenizer(), use_color=False)++ assert capsys.readouterr().out == "hello\n"+++def test_content_sequence_visualize_merges_audio_and_semantic_tokens(+ monkeypatch, capsys+):+ sequence = ContentSequence()+ monkeypatch.setattr(+ sequence,+ "encode",+ lambda *args, **kwargs: encoded(+ [100, 101, 200, 200, 200, 1],+ [-100, -100, -100, -100, 1, 1],+ ),+ )++ sequence.visualize(+ FakeTokenizer(),+ merge_semantic_tokens=True,+ merge_audio_tokens=True,+ use_color=False,+ )++ assert (+ capsys.readouterr().out+ == "[<|semantic|>x2][<|audio_pad|>x2][<|audio_pad|>x1]hello\n"+ )+++def test_conversation_visualize_forwards_options(monkeypatch):+ sequence = ContentSequence()+ monkeypatch.setattr(Conversation, "_build_content_sequence", lambda self: sequence)+ received = {}++ def visualize(tokenizer, **kwargs):+ received.update(kwargs)++ monkeypatch.setattr(sequence, "visualize", visualize)++ Conversation().visualize(+ FakeTokenizer(),+ merge_semantic_tokens=True,+ merge_audio_tokens=True,+ use_color=False,+ )++ assert received == {+ "ignore_loss_tokens": [],+ "merge_semantic_tokens": True,+ "merge_audio_tokens": True,+ "use_color": False,+ }