SOTA Open Source TTS
import base64
import os
import queue
from dataclasses import dataclass
from typing import Literal
import torch
from pydantic import BaseModel, Field, conint, model_validator
from pydantic.functional_validators import SkipValidation
from typing_extensions import Annotated
from fish_speech.content_sequence import TextPart, VQPart
class ServeVQPart(BaseModel):
type: Literal["vq"] = "vq"
codes: SkipValidation[list[list[int]]]
class ServeTextPart(BaseModel):
type: Literal["text"] = "text"
text: str
class ServeAudioPart(BaseModel):
type: Literal["audio"] = "audio"
audio: bytes
class ServeVQGANEncodeRequest(BaseModel):
# The audio here should be in wav, mp3, etc
audios: list[bytes]
class ServeVQGANEncodeResponse(BaseModel):
tokens: SkipValidation[list[list[list[int]]]]
class ServeVQGANDecodeRequest(BaseModel):
tokens: SkipValidation[list[list[list[int]]]]
class ServeVQGANDecodeResponse(BaseModel):
# The audio here should be in PCM float16 format
audios: list[bytes]
class ServeReferenceAudio(BaseModel):
audio: bytes
text: str
# The speaker this reference voices. A transcript that does not start with a
# <|speaker:N|> tag gets this one; None keeps the model's default (S2-Pro: the
# reference's position, Drama 3: speaker 0).
speaker: Annotated[int, Field(ge=0, le=15, strict=True)] | None = None
@model_validator(mode="before")
def decode_audio(cls, values):
audio = values.get("audio")
if (
isinstance(audio, str) and len(audio) > 255
): # Check if audio is a string (Base64)
try:
values["audio"] = base64.b64decode(audio)
except Exception:
# If the audio is not a valid base64 string, we will just ignore it and let the server handle it
pass
return values
def __repr__(self) -> str:
return (
f"ServeReferenceAudio(text={self.text!r}, speaker={self.speaker}, "
f"audio_size={len(self.audio)})"
)
class ServeTTSRequest(BaseModel):
text: str
chunk_length: Annotated[int, conint(ge=100, le=1000, strict=True)] = 200
# Drama 3 only: generate the whole text in one pass instead of ~300-byte chunks.
single_chunk: bool = False
# Audio format
format: Literal["wav", "pcm", "mp3", "opus"] = "wav"
# Latency mode (used by api.fish.audio; "normal" or "balanced")
latency: Literal["normal", "balanced"] = "normal"
# References audios for in-context learning
references: list[ServeReferenceAudio] = []
# Reference id
# For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
# Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
reference_id: str | None = None
seed: int | None = None
use_memory_cache: Literal["on", "off"] = "off"
# Normalize text for en & zh, this increase stability for numbers
normalize: bool = True
# not usually used below
streaming: bool = False
# Sampling: None takes the official Fish Audio SDK's default (max_new_tokens
# 1024 per text chunk; top_p 0.7; repetition_penalty 1.2; temperature 0.7).
# max_new_tokens <= 0 means the model's own cap (S2-Pro: the context length;
# Drama 3: 600 frames per chunk, or 2048 with single_chunk). repetition_penalty is
# accepted for compatibility and ignored: both models use repetition-aware
# sampling.
max_new_tokens: int | None = None
top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] | None = None
repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] | None = (
None
)
temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] | None = None
class Config:
# Allow arbitrary types for pytorch related types
arbitrary_types_allowed = True
class AddReferenceRequest(BaseModel):
id: str = Field(..., min_length=1, max_length=255, pattern=r"^[a-zA-Z0-9\-_ ]+$")
audio: bytes
text: str = Field(..., min_length=1)
class AddReferenceResponse(BaseModel):
success: bool
message: str
reference_id: str
class ListReferencesResponse(BaseModel):
success: bool
reference_ids: list[str]
message: str = "Success"
class DeleteReferenceResponse(BaseModel):
success: bool
message: str
reference_id: str
class UpdateReferenceResponse(BaseModel):
success: bool
message: str
old_reference_id: str
new_reference_id: str