The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(FishAudio): implement basic nodes (#15612)
Alexander Piskun7d39997e9f38
diff --git a/comfy_api_nodes/apis/fishaudio.py b/comfy_api_nodes/apis/fishaudio.pynew file mode 100644index 000000000..f615d3d24--- /dev/null+++ b/comfy_api_nodes/apis/fishaudio.py@@ -0,0 +1,49 @@+from pydantic import BaseModel, Field+++class FishAudioProsody(BaseModel):+ speed: float = Field(1.0, description="Speaking rate multiplier, 0.5-2.0")+ volume: float = Field(0.0, description="Volume adjustment in decibels")+++class FishAudioTTSRequest(BaseModel):+ text: str = Field(..., description="Text to synthesize")+ reference_id: str | list[str] | None = Field(None, description="Voice model ID or list of IDs")+ temperature: float = Field(0.7, description="Expressiveness, 0-1")+ top_p: float = Field(0.7, description="Nucleus sampling diversity, (0, 1]")+ prosody: FishAudioProsody = Field(..., description="Speed and volume adjustments")+ normalize: bool = Field(True, description="Normalize numbers and text for English and Chinese")+ format: str = Field("wav", description="Output audio format")+++class FishAudioASRRequest(BaseModel):+ language: str | None = Field(None, description="Optional ISO 639-1 language hint")+ ignore_timestamps: bool = Field(True, description="Skip precise timestamp computation")+++class FishAudioASRSegment(BaseModel):+ text: str | None = Field(None, description="Segment text")+ start: float | None = Field(None, description="Segment start time in seconds")+ end: float | None = Field(None, description="Segment end time in seconds")+++class FishAudioASRResponse(BaseModel):+ text: str | None = Field(None, description="Transcribed text")+ duration: float | None = Field(None, description="Audio duration in seconds")+ segments: list[FishAudioASRSegment] | None = Field(None, description="Timestamped transcript segments")+ language_code: str | None = Field(None, description="Detected language as ISO 639-1 code")+ language: str | None = Field(None, description="Detected language display name")+++class FishAudioCreateModelRequest(BaseModel):+ type: str = Field("tts", description="Model type")+ title: str = Field(..., description="Voice model name")+ train_mode: str = Field("fast", description="Training mode; fast is instantly available")+ visibility: str = Field("private", description="Model visibility")+ enhance_audio_quality: bool = Field(..., description="Enhance reference audio quality")+++class FishAudioCreateModelResponse(BaseModel):+ id: str = Field(..., alias="_id", description="Voice model ID for use as reference_id")+ state: str | None = Field(None, description="Training state")+ visibility: str | None = Field(None, description="Model visibility")diff --git a/comfy_api_nodes/nodes_fishaudio.py b/comfy_api_nodes/nodes_fishaudio.pynew file mode 100644index 000000000..d40fc2740--- /dev/null+++ b/comfy_api_nodes/nodes_fishaudio.py@@ -0,0 +1,454 @@+import json+import re+import uuid++from typing_extensions import override++from comfy_api.latest import IO, ComfyExtension, Input+from comfy_api_nodes.apis.fishaudio import (+ FishAudioASRRequest,+ FishAudioASRResponse,+ FishAudioCreateModelRequest,+ FishAudioCreateModelResponse,+ FishAudioProsody,+ FishAudioTTSRequest,+)+from comfy_api_nodes.util import (+ ApiEndpoint,+ audio_bytes_to_audio_input,+ audio_ndarray_to_bytesio,+ audio_tensor_to_contiguous_ndarray,+ sync_op,+ sync_op_raw,+ validate_string,+)++FISHAUDIO_VOICE = "FISHAUDIO_VOICE"++FISHAUDIO_VOICES = [+ ("802e3bc2b27e49c2995d23ef70e6ac89", "Energetic Male (en)"),+ ("b545c585f631496c914815291da4e893", "Friendly Women (en)"),+ ("933563129e564b19a115bedd57b7406a", "Sarah (en)"),+ ("8d21b053e2804e2a890e1cf62f267b6f", "Verity (en)"),+ ("f48d143a59a946ab87c0130fd081f349", "Polo (en)"),+ ("bf322df2096a46f18c579d0baa36f41d", "Adrian (en)"),+ ("98655a12fa944e26b274c535e5e03842", "E-girl (en)"),+ ("0327fdb5da9e4fd782899a8058c8ae2b", "Narrator (en)"),+ ("5212eb29e500460391d03af42af6552e", "Warm Conversational Voice (en)"),+ ("5c8dc6a69c0b4edfb32634db6384bf34", "Warm Storyteller (en)"),+ ("7a18a1851d2649108c48ec9f2c80eb2c", "Dramatic Character Male (en)"),+ ("59cb5986671546eaa6ca8ae6f29f6d22", "News Narrator (zh)"),+ ("bf6c479f5a384b8d857310030035824b", "Lively Female (zh)"),+ ("faccba1a8ac54016bcfc02761285e67f", "Gentle Female (zh)"),+ ("5161d41404314212af1254556477c17d", "Energetic Female (ja)"),+ ("0089dce5fefb4c6ba9b9f2f0debe1ddc", "Calm Female (ja)"),+ ("45c5d3723c9c42f598e4776dcfd5f02d", "Calm Male (ja)"),+]++FISHAUDIO_VOICE_MAP = {label: voice_id for voice_id, label in FISHAUDIO_VOICES}++MAX_REFERENCE_AUDIO_SECONDS = 270+++def _rewrite_voice_tags(text: str, voice_count: int) -> tuple[str, set[int]]:+ referenced: set[int] = set()++ def repl(match: re.Match) -> str:+ index = int(match.group(1))+ if index < 1 or index > voice_count:+ raise ValueError(+ f"@Voice{index} does not match any connected voice ({voice_count} connected)."+ )+ referenced.add(index)+ return f"<|speaker:{index - 1}|>"++ rewritten = re.sub(r"(?<!\S)@voice([0-9]+)\b", repl, text, flags=re.IGNORECASE)+ return rewritten, referenced+++def _tts_option_inputs() -> list:+ return [+ IO.Float.Input(+ "temperature",+ default=0.7,+ min=0.0,+ max=1.0,+ step=0.01,+ display_mode=IO.NumberDisplay.slider,+ tooltip="Expressiveness. Higher values are more varied, lower values are more consistent.",+ ),+ IO.Float.Input(+ "top_p",+ default=0.7,+ min=0.01,+ max=1.0,+ step=0.01,+ display_mode=IO.NumberDisplay.slider,+ tooltip="Diversity via nucleus sampling.",+ ),+ IO.Float.Input(+ "speed",+ default=1.0,+ min=0.5,+ max=2.0,+ step=0.01,+ display_mode=IO.NumberDisplay.slider,+ tooltip="Speaking rate. 1.0 is normal, <1.0 slower, >1.0 faster.",+ ),+ IO.Float.Input(+ "volume",+ default=0.0,+ min=-10.0,+ max=10.0,+ step=0.5,+ display_mode=IO.NumberDisplay.slider,+ tooltip="Volume adjustment in decibels. 0 is no change.",+ ),+ IO.Boolean.Input(+ "normalize",+ default=True,+ tooltip="Normalize numbers and text for English and Chinese, "+ "improving stability for numbers and dates.",+ ),+ ]+++def _multi_speaker_inputs() -> list:+ return [+ IO.Autogrow.Input(+ "voices",+ template=IO.Autogrow.TemplatePrefix(+ IO.Custom(FISHAUDIO_VOICE).Input("voice"),+ prefix="voice",+ min=0,+ max=5,+ ),+ tooltip="Voices for synthesis. Leave empty for the default voice. "+ "With two or more voices, mark speaker changes in the text with @Voice1, @Voice2, etc.",+ ),+ *_tts_option_inputs(),+ ]+++class FishAudioVoiceSelector(IO.ComfyNode):+ @classmethod+ def define_schema(cls) -> IO.Schema:+ return IO.Schema(+ node_id="FishAudioVoiceSelector",+ display_name="Fish Audio Voice Selector",+ category="partner/audio/Fish Audio",+ description="Select a voice from the Fish Audio library for text-to-speech generation.",+ inputs=[+ IO.DynamicCombo.Input(+ "voice",+ options=[+ *(IO.DynamicCombo.Option(label, []) for _, label in FISHAUDIO_VOICES),+ IO.DynamicCombo.Option(+ "custom",+ [+ IO.String.Input(+ "voice_id",+ default="",+ tooltip="Voice model ID from fish.audio, e.g. the ID in "+ "https://fish.audio/m/<id>/.",+ ),+ ],+ ),+ ],+ tooltip="Choose a voice, or 'custom' to enter any fish.audio voice model ID.",+ ),+ ],+ outputs=[+ IO.Custom(FISHAUDIO_VOICE).Output(display_name="voice"),+ ],+ is_api_node=False,+ )++ @classmethod+ def execute(cls, voice: dict) -> IO.NodeOutput:+ selected = voice["voice"]+ if selected == "custom":+ voice_id = voice["voice_id"].strip()+ if not voice_id:+ raise ValueError("Custom voice ID is empty.")+ return IO.NodeOutput(voice_id)+ voice_id = FISHAUDIO_VOICE_MAP.get(selected)+ if not voice_id:+ raise ValueError(f"Unknown voice: {selected}")+ return IO.NodeOutput(voice_id)+++class FishAudioTextToSpeech(IO.ComfyNode):+ @classmethod+ def define_schema(cls) -> IO.Schema:+ return IO.Schema(+ node_id="FishAudioTextToSpeech",+ display_name="Fish Audio Text to Speech",+ category="partner/audio/Fish Audio",+ description="Convert text to speech. Supports emotion cues in the text "+ "([happy], [whispering] on s2.1-pro; (happy) on s1) and multi-speaker dialogue "+ "via @Voice1/@Voice2 tags with multiple connected voices.",+ inputs=[+ IO.String.Input(+ "text",+ multiline=True,+ default="",+ tooltip="The text to convert to speech. With two or more voices connected, "+ "mark speaker changes with @Voice1, @Voice2, etc.",+ ),+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option("s2.1-pro", _multi_speaker_inputs()),+ IO.DynamicCombo.Option(+ "s1",+ [+ IO.Custom(FISHAUDIO_VOICE).Input(+ "voice",+ optional=True,+ tooltip="Voice for synthesis. Leave unconnected for the default voice.",+ ),+ *_tts_option_inputs(),+ ],+ ),+ ],+ tooltip="Model to use for text-to-speech.",+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=2147483647,+ display_mode=IO.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Seed controls whether the node should re-run; "+ "results are non-deterministic regardless of seed.",+ ),+ ],+ outputs=[+ IO.Audio.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(widgets=["text"]),+ expr="""+ (+ $t := widgets.text;+ $type($t) = "string"+ ? (+ $bytes := $length($t) + 2 * $count($match($t, /[^\\x00-\\x7F]/));+ {"type":"usd","usd": $bytes * 21.45 / 1000000, "format":{"approximate":true}}+ )+ : {"type":"usd","usd": 0.02145, "format":{"approximate":true, "suffix":"/1K bytes"}}+ )+ """,+ ),+ )++ @classmethod+ async def execute(+ cls,+ text: str,+ model: dict,+ seed: int,+ ) -> IO.NodeOutput:+ validate_string(text, field_name="text", min_length=1)+ model_name = model["model"]+ if model_name == "s1":+ voices = [model["voice"]] if model.get("voice") else []+ else:+ voices = [model["voices"][key] for key in model["voices"]]+ rewritten, referenced = _rewrite_voice_tags(text, len(voices))+ if len(voices) >= 2:+ missing = [i for i in range(1, len(voices) + 1) if i not in referenced]+ if missing:+ raise ValueError(+ "With multiple voices, the text must mark speaker changes with tags for "+ "each connected voice; missing: " + ", ".join(f"@Voice{i}" for i in missing)+ )+ reference_id: str | list[str] | None = None+ if len(voices) == 1:+ reference_id = voices[0]+ elif voices:+ reference_id = voices+ request = FishAudioTTSRequest(+ text=rewritten,+ reference_id=reference_id,+ temperature=model["temperature"],+ top_p=model["top_p"],+ prosody=FishAudioProsody(speed=model["speed"], volume=model["volume"]),+ normalize=model["normalize"],+ )+ response = await sync_op_raw(+ cls,+ ApiEndpoint(+ path="/proxy/fishaudio/v1/tts",+ method="POST",+ headers={"model": model_name},+ ),+ data=request,+ as_binary=True,+ )+ return IO.NodeOutput(audio_bytes_to_audio_input(response))+++class FishAudioSpeechToText(IO.ComfyNode):+ @classmethod+ def define_schema(cls) -> IO.Schema:+ return IO.Schema(+ node_id="FishAudioSpeechToText",+ display_name="Fish Audio Speech to Text",+ category="partner/audio/Fish Audio",+ description="Transcribe audio to text with automatic language detection.",+ inputs=[+ IO.Audio.Input(+ "audio",+ tooltip="Audio to transcribe.",+ ),+ IO.String.Input(+ "language",+ default="",+ tooltip="ISO 639-1 language hint (e.g. 'en', 'zh'). "+ "The language is auto-detected regardless.",+ ),+ IO.Boolean.Input(+ "precise_timestamps",+ default=False,+ tooltip="Return word-level timestamped segments.",+ ),+ ],+ outputs=[+ IO.String.Output(id="text", display_name="text"),+ IO.String.Output(id="language_code", display_name="language_code"),+ IO.String.Output(id="segments_json", display_name="segments_json"),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ expr="""{"type":"usd","usd":0.00858,"format":{"approximate":true,"suffix":"/minute"}}""",+ ),+ )++ @classmethod+ async def execute(+ cls,+ audio: Input.Audio,+ language: str,+ precise_timestamps: bool,+ ) -> IO.NodeOutput:+ audio_data_np = audio_tensor_to_contiguous_ndarray(audio["waveform"])+ audio_bytes_io = audio_ndarray_to_bytesio(audio_data_np, audio["sample_rate"], "mp4", "aac")+ response = await sync_op(+ cls,+ ApiEndpoint(path="/proxy/fishaudio/v1/asr", method="POST"),+ response_model=FishAudioASRResponse,+ data=FishAudioASRRequest(+ language=language.strip() or None,+ ignore_timestamps=not precise_timestamps,+ ),+ files={"audio": ("audio.mp4", audio_bytes_io, "audio/mp4")},+ content_type="multipart/form-data",+ )+ segments_json = json.dumps(+ [s.model_dump(exclude_none=True) for s in (response.segments or [])],+ indent=2,+ )+ return IO.NodeOutput(response.text or "", response.language_code or "", segments_json)+++class FishAudioInstantVoiceClone(IO.ComfyNode):+ @classmethod+ def define_schema(cls) -> IO.Schema:+ return IO.Schema(+ node_id="FishAudioInstantVoiceClone",+ display_name="Fish Audio Instant Voice Clone",+ category="partner/audio/Fish Audio",+ description="Create a private cloned voice from audio samples, instantly usable "+ "for text-to-speech. Provide 1-20 recordings, 10-30 seconds each recommended, "+ "under 270 seconds in total.",+ inputs=[+ IO.Autogrow.Input(+ "files",+ template=IO.Autogrow.TemplatePrefix(+ IO.Audio.Input("audio"),+ prefix="audio",+ min=1,+ max=20,+ ),+ tooltip="Audio recordings for voice cloning.",+ ),+ IO.Boolean.Input(+ "enhance_audio_quality",+ default=True,+ tooltip="Enhance reference audio quality before training.",+ ),+ ],+ outputs=[+ IO.Custom(FISHAUDIO_VOICE).Output(display_name="voice"),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(expr="""{"type":"usd","usd":0}"""),+ )++ @classmethod+ async def execute(+ cls,+ files: IO.Autogrow.Type,+ enhance_audio_quality: bool,+ ) -> IO.NodeOutput:+ total_seconds = 0.0+ for key in files:+ audio = files[key]+ total_seconds += audio["waveform"].shape[-1] / audio["sample_rate"]+ if total_seconds >= MAX_REFERENCE_AUDIO_SECONDS:+ raise ValueError(+ f"Total reference audio is {total_seconds:.0f} seconds; "+ f"it must be under {MAX_REFERENCE_AUDIO_SECONDS} seconds."+ )+ file_tuples: list[tuple[str, tuple[str, bytes, str]]] = []+ for key in files:+ audio = files[key]+ audio_data_np = audio_tensor_to_contiguous_ndarray(audio["waveform"])+ audio_bytes_io = audio_ndarray_to_bytesio(audio_data_np, audio["sample_rate"], "mp4", "aac")+ file_tuples.append(("voices", (f"{key}.mp4", audio_bytes_io.getvalue(), "audio/mp4")))+ response = await sync_op(+ cls,+ ApiEndpoint(path="/proxy/fishaudio/model", method="POST"),+ response_model=FishAudioCreateModelResponse,+ data=FishAudioCreateModelRequest(+ title=str(uuid.uuid4()),+ enhance_audio_quality=enhance_audio_quality,+ ),+ files=file_tuples,+ content_type="multipart/form-data",+ )+ return IO.NodeOutput(response.id)+++class FishAudioExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[IO.ComfyNode]]:+ return [+ FishAudioVoiceSelector,+ FishAudioTextToSpeech,+ FishAudioSpeechToText,+ FishAudioInstantVoiceClone,+ ]+++async def comfy_entrypoint() -> FishAudioExtension:+ return FishAudioExtension()