Skip to content

SOTA Open Source TTS

fix: honor conversation visualization options (#1331)

Avinash Gola44a034dc3622
diff --git a/fish_speech/content_sequence.py b/fish_speech/content_sequence.py
index c29f8c0..8ec8136 100644
--- a/fish_speech/content_sequence.py
+++ b/fish_speech/content_sequence.py
@@ -5,6 +5,7 @@ import numpy as np
import torch
from fish_speech.tokenizer import (
+ AUDIO_EMBED_TOKEN,
IM_END_TOKEN,
MODALITY_TOKENS,
FishTokenizer,
@@ -328,6 +329,8 @@ class ContentSequence:
tokenizer: FishTokenizer,
ignore_loss_tokens: list[str] = [],
merge_semantic_tokens: bool = False,
+ merge_audio_tokens: bool = False,
+ use_color: bool = True,
):
"""
Visualize the encoded sequence with color-coded tokens.
@@ -349,41 +352,69 @@ class ContentSequence:
def print_in_blue(x):
nonlocal blue_idx
- color = colors["blue"] if blue_idx % 2 == 0 else colors["cyan"]
- print(f"{color}{x}\033[0m", end="")
+ if use_color:
+ color = colors["blue"] if blue_idx % 2 == 0 else colors["cyan"]
+ print(f"{color}{x}\033[0m", end="")
+ else:
+ print(x, end="")
blue_idx += 1
def print_in_green(x):
nonlocal green_idx
- color = colors["green"] if green_idx % 2 == 0 else colors["dark_green"]
- print(f"{color}{x}\033[0m", end="")
+ if use_color:
+ color = colors["green"] if green_idx % 2 == 0 else colors["dark_green"]
+ print(f"{color}{x}\033[0m", end="")
+ else:
+ print(x, end="")
green_idx += 1
- def print_semantic_token(x, count):
- val = f"[<|semantic|>x{count}]"
- if x == -100:
+ def print_merged_token(name: str, label: int, count: int):
+ val = f"[<|{name}|>x{count}]"
+ if label == -100:
print_in_green(val)
else:
print_in_blue(val)
- count_semantic_tokens = 0
- semantic_label = None
+ audio_token_id = (
+ tokenizer.get_token_id(AUDIO_EMBED_TOKEN) if merge_audio_tokens else None
+ )
+ merged_kind = None
+ merged_label = None
+ merged_count = 0
+
+ def flush_merged_tokens():
+ nonlocal merged_kind, merged_label, merged_count
+ if merged_kind is not None:
+ assert merged_label is not None
+ print_merged_token(merged_kind, merged_label, merged_count)
+ merged_kind = None
+ merged_label = None
+ merged_count = 0
for tok, lab in zip(encoded.tokens, encoded.labels):
token_id = int(tok.item())
+ label = int(lab.item())
+
+ token_kind = None
+ if (
+ merge_semantic_tokens
+ and tokenizer.semantic_begin_id <= token_id <= tokenizer.semantic_end_id
+ ):
+ token_kind = "semantic"
+ elif merge_audio_tokens and token_id == audio_token_id:
+ token_kind = "audio_pad"
+
+ if token_kind is not None:
+ if token_kind == merged_kind and label == merged_label:
+ merged_count += 1
+ else:
+ flush_merged_tokens()
+ merged_kind = token_kind
+ merged_label = label
+ merged_count = 1
+ continue
- if merge_semantic_tokens:
- if (
- tokenizer.semantic_begin_id <= token_id <= tokenizer.semantic_end_id
- and (semantic_label is None or semantic_label == lab)
- ):
- count_semantic_tokens += 1
- semantic_label = lab
- continue
- elif count_semantic_tokens > 0:
- print_semantic_token(semantic_label, count_semantic_tokens)
- count_semantic_tokens = 0
- semantic_label = None
+ flush_merged_tokens()
# Use HF decode
val = tokenizer.decode([token_id])
@@ -392,12 +423,11 @@ class ContentSequence:
if not val:
val = f"<{token_id}>"
- if lab == -100:
+ if label == -100:
print_in_green(val)
else:
print_in_blue(val)
- if merge_semantic_tokens and count_semantic_tokens > 0:
- print_semantic_token(semantic_label, count_semantic_tokens)
+ flush_merged_tokens()
print()
diff --git a/fish_speech/conversation.py b/fish_speech/conversation.py
index d0fa5c2..1629202 100644
--- a/fish_speech/conversation.py
+++ b/fish_speech/conversation.py
@@ -120,6 +120,8 @@ class Conversation:
tokenizer,
ignore_loss_tokens=ignore_loss_tokens,
merge_semantic_tokens=merge_semantic_tokens,
+ merge_audio_tokens=merge_audio_tokens,
+ use_color=use_color,
)
def append(self: "Conversation", message: Message):
diff --git a/tests/test_visualization.py b/tests/test_visualization.py
new file mode 100644
index 0000000..e70dd2d
--- /dev/null
+++ b/tests/test_visualization.py
@@ -0,0 +1,95 @@
+from types import SimpleNamespace
+
+from fish_speech.content_sequence import ContentSequence
+from fish_speech.conversation import Conversation
+from fish_speech.tokenizer import AUDIO_EMBED_TOKEN
+
+
+class FakeTokenizer:
+ semantic_begin_id = 100
+ semantic_end_id = 199
+ audio_token_id = 200
+
+ def get_token_id(self, token):
+ assert token == AUDIO_EMBED_TOKEN
+ return self.audio_token_id
+
+ def decode(self, tokens):
+ return {
+ 1: "hello",
+ self.audio_token_id: AUDIO_EMBED_TOKEN,
+ }.get(tokens[0], f"<|semantic:{tokens[0] - self.semantic_begin_id}|>")
+
+
+class FakeScalar(int):
+ def item(self):
+ return int(self)
+
+
+def encoded(tokens, labels):
+ return SimpleNamespace(
+ tokens=[FakeScalar(token) for token in tokens],
+ labels=[FakeScalar(label) for label in labels],
+ )
+
+
+def test_content_sequence_visualize_without_color(monkeypatch, capsys):
+ sequence = ContentSequence()
+ monkeypatch.setattr(
+ sequence, "encode", lambda *args, **kwargs: encoded([1], [-100])
+ )
+
+ sequence.visualize(FakeTokenizer(), use_color=False)
+
+ assert capsys.readouterr().out == "hello\n"
+
+
+def test_content_sequence_visualize_merges_audio_and_semantic_tokens(
+ monkeypatch, capsys
+):
+ sequence = ContentSequence()
+ monkeypatch.setattr(
+ sequence,
+ "encode",
+ lambda *args, **kwargs: encoded(
+ [100, 101, 200, 200, 200, 1],
+ [-100, -100, -100, -100, 1, 1],
+ ),
+ )
+
+ sequence.visualize(
+ FakeTokenizer(),
+ merge_semantic_tokens=True,
+ merge_audio_tokens=True,
+ use_color=False,
+ )
+
+ assert (
+ capsys.readouterr().out
+ == "[<|semantic|>x2][<|audio_pad|>x2][<|audio_pad|>x1]hello\n"
+ )
+
+
+def test_conversation_visualize_forwards_options(monkeypatch):
+ sequence = ContentSequence()
+ monkeypatch.setattr(Conversation, "_build_content_sequence", lambda self: sequence)
+ received = {}
+
+ def visualize(tokenizer, **kwargs):
+ received.update(kwargs)
+
+ monkeypatch.setattr(sequence, "visualize", visualize)
+
+ Conversation().visualize(
+ FakeTokenizer(),
+ merge_semantic_tokens=True,
+ merge_audio_tokens=True,
+ use_color=False,
+ )
+
+ assert received == {
+ "ignore_loss_tokens": [],
+ "merge_semantic_tokens": True,
+ "merge_audio_tokens": True,
+ "use_color": False,
+ }