Skip to content

SOTA Open Source TTS

4.4 KB
Raw Blame
import base64
import os
import queue
from dataclasses import dataclass
from typing import Literal

import torch
from pydantic import BaseModel, Field, conint, model_validator
from pydantic.functional_validators import SkipValidation
from typing_extensions import Annotated

from fish_speech.content_sequence import TextPart, VQPart


class ServeVQPart(BaseModel):
    type: Literal["vq"] = "vq"
    codes: SkipValidation[list[list[int]]]


class ServeTextPart(BaseModel):
    type: Literal["text"] = "text"
    text: str


class ServeAudioPart(BaseModel):
    type: Literal["audio"] = "audio"
    audio: bytes


class ServeVQGANEncodeRequest(BaseModel):
    # The audio here should be in wav, mp3, etc
    audios: list[bytes]


class ServeVQGANEncodeResponse(BaseModel):
    tokens: SkipValidation[list[list[list[int]]]]


class ServeVQGANDecodeRequest(BaseModel):
    tokens: SkipValidation[list[list[list[int]]]]


class ServeVQGANDecodeResponse(BaseModel):
    # The audio here should be in PCM float16 format
    audios: list[bytes]


class ServeReferenceAudio(BaseModel):
    audio: bytes
    text: str
    # The speaker this reference voices. A transcript that does not start with a
    # <|speaker:N|> tag gets this one; None keeps the model's default (S2-Pro: the
    # reference's position, Drama 3: speaker 0).
    speaker: Annotated[int, Field(ge=0, le=15, strict=True)] | None = None

    @model_validator(mode="before")
    def decode_audio(cls, values):
        audio = values.get("audio")
        if (
            isinstance(audio, str) and len(audio) > 255
        ):  # Check if audio is a string (Base64)
            try:
                values["audio"] = base64.b64decode(audio)
            except Exception:
                # If the audio is not a valid base64 string, we will just ignore it and let the server handle it
                pass
        return values

    def __repr__(self) -> str:
        return (
            f"ServeReferenceAudio(text={self.text!r}, speaker={self.speaker}, "
            f"audio_size={len(self.audio)})"
        )


class ServeTTSRequest(BaseModel):
    text: str
    chunk_length: Annotated[int, conint(ge=100, le=1000, strict=True)] = 200
    # Drama 3 only: generate the whole text in one pass instead of ~300-byte chunks.
    single_chunk: bool = False
    # Audio format
    format: Literal["wav", "pcm", "mp3", "opus"] = "wav"
    # Latency mode (used by api.fish.audio; "normal" or "balanced")
    latency: Literal["normal", "balanced"] = "normal"
    # References audios for in-context learning
    references: list[ServeReferenceAudio] = []
    # Reference id
    # For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
    # Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
    reference_id: str | None = None
    seed: int | None = None
    use_memory_cache: Literal["on", "off"] = "off"
    # Normalize text for en & zh, this increase stability for numbers
    normalize: bool = True
    # not usually used below
    streaming: bool = False
    # Sampling: None takes the official Fish Audio SDK's default (max_new_tokens
    # 1024 per text chunk; top_p 0.7; repetition_penalty 1.2; temperature 0.7).
    # max_new_tokens <= 0 means the model's own cap (S2-Pro: the context length;
    # Drama 3: 600 frames per chunk, or 2048 with single_chunk). repetition_penalty is
    # accepted for compatibility and ignored: both models use repetition-aware
    # sampling.
    max_new_tokens: int | None = None
    top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] | None = None
    repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] | None = (
        None
    )
    temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] | None = None

    class Config:
        # Allow arbitrary types for pytorch related types
        arbitrary_types_allowed = True


class AddReferenceRequest(BaseModel):
    id: str = Field(..., min_length=1, max_length=255, pattern=r"^[a-zA-Z0-9\-_ ]+$")
    audio: bytes
    text: str = Field(..., min_length=1)


class AddReferenceResponse(BaseModel):
    success: bool
    message: str
    reference_id: str


class ListReferencesResponse(BaseModel):
    success: bool
    reference_ids: list[str]
    message: str = "Success"


class DeleteReferenceResponse(BaseModel):
    success: bool
    message: str
    reference_id: str


class UpdateReferenceResponse(BaseModel):
    success: bool
    message: str
    old_reference_id: str
    new_reference_id: str